From 239478b5e2fceb5cc297fa81ca44dd5c8066c42c Mon Sep 17 00:00:00 2001 From: zacYL Date: Mon, 3 Jul 2023 10:33:56 +0800 Subject: [PATCH 01/41] =?UTF-8?q?bug:=20=E6=B7=BB=E5=8A=A0=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E9=95=9C=E5=83=8F=E4=BB=93=E5=BA=93package?= =?UTF-8?q?=E6=8B=89=E5=8F=96=E6=89=A9=E5=B1=95=E7=82=B9=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ImagePackageInfoPullExtension.kt | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt new file mode 100644 index 0000000000..9d18bd98c6 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.extension + +import com.tencent.devops.plugin.api.ExtensionPoint + +/** + * 拉取并存储第三方仓库下的镜像对应package信息扩展点 + */ +interface ImagePackageInfoPullExtension: ExtensionPoint { + + /** + * 拉取第三方镜像仓库package信息,并存储到对应仓库中 + */ + fun queryAndCreateDockerPackageInfo( + projectId: String, + repoName:String, + remoteUrl: String, + userName: String? = null, + password: String? = null + ) +} From 7cbceedfbc9e4e0d207a9f26321983b8257d5b26 Mon Sep 17 00:00:00 2001 From: zacYL Date: Mon, 3 Jul 2023 19:43:48 +0800 Subject: [PATCH 02/41] =?UTF-8?q?bug:=20=E9=95=9C=E5=83=8F=E4=BB=93?= =?UTF-8?q?=E5=BA=93=E6=94=AF=E6=8C=81remote=E4=BB=A3=E7=90=86=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/api/constant/AuthenticationKeys.kt | 34 +++ .../authentication}/AuthenticationProperty.kt | 2 +- .../common/api/util/AuthenticationUtil.kt | 74 +++++ .../common/artifact/util/http/UrlFormatter.kt | 83 ++++++ .../artifact/util/http/UrlFormatterTest.kt | 176 ++++++++++++ .../tencent/bkrepo/job/batch/RepoInitJob.kt | 17 +- .../bkrepo/job/batch/RepoRefreshJob.kt | 15 +- .../bkrepo/job/batch/base/DefaultRepoJob.kt | 1 + .../config/properties/RepoJobProperties.kt | 2 +- .../com/tencent/bkrepo/oci/api/OciClient.kt | 9 +- .../bkrepo/oci/constant/OciConstants.kt | 8 +- .../bkrepo/oci/constant/OciMessageCode.kt | 4 +- .../ImagePackageInfoPullExtension.kt | 6 +- .../oci/extension/ImagePackagePullContext.kt | 38 +++ .../oci/pojo/remote/RemoteRequestProperty.kt | 39 +++ .../repository/OciRegistryRemoteRepository.kt | 258 ++++++++++++------ .../service/OciPackageController.kt | 9 +- .../bkrepo/oci/listener/base/EventExecutor.kt | 13 +- .../consumer/RemoteImageRepoEventConsumer.kt | 73 +++++ .../bkrepo/oci/service/OciOperationService.kt | 5 + .../service/impl/OciOperationServiceImpl.kt | 55 +++- .../resources/i18n/messages_en.properties | 4 +- .../resources/i18n/messages_zh_CN.properties | 4 +- .../resources/i18n/messages_zh_TW.properties | 4 +- .../bkrepo/replication/constant/Constants.kt | 3 - .../handler/ArtifactReplicationHandler.kt | 6 +- .../type/oci/OciAuthorizationService.kt | 53 +--- .../service/impl/RemoteNodeServiceImpl.kt | 2 +- .../bkrepo/replication/util/HttpUtils.kt | 62 +---- 29 files changed, 832 insertions(+), 227 deletions(-) create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt rename src/backend/{replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote => common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication}/AuthenticationProperty.kt (96%) create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt create mode 100644 src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt create mode 100644 src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt new file mode 100644 index 0000000000..070361adef --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.constant + +object AuthenticationKeys { + const val BEARER_REALM = "Bearer realm" + const val SERVICE = "service" + const val SCOPE = "scope" +} \ No newline at end of file diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt similarity index 96% rename from src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt rename to src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt index 47e4268a7f..5d10b119fe 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.pojo.remote +package com.tencent.bkrepo.common.api.pojo.authentication /** * 返回头中的WWW_AUTHENTICATE字段包含的属性 diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt new file mode 100644 index 0000000000..d97bdb2e44 --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.util + +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.BEARER_REALM +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SCOPE +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SERVICE +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.pojo.authentication.AuthenticationProperty + +object AuthenticationUtil { + + /** + * 解析返回头中的WWW_AUTHENTICATE字段， 只针对为Bearer realm + */ + fun parseWWWAuthenticateHeader(wwwAuthenticate: String, scope: String?): AuthenticationProperty? { + val map: MutableMap = mutableMapOf() + return try { + val params = wwwAuthenticate.split("\",") + params.forEach { + val param = it.split(Regex("="),2) + val name = param.first() + val value = param.last().replace("\"", "") + map[name] = value + } + AuthenticationProperty( + authUrl = map[BEARER_REALM]!!, + service = map[SERVICE]!!, + scope = scope + ) + } catch (e: Exception) { + null + } + } + + fun buildAuthenticationUrl(property: AuthenticationProperty, userName: String?): String? { + if (property.authUrl.isBlank()) return null + var result = if (property.authUrl.contains(StringPool.QUESTION)) { + "${property.authUrl}&$SERVICE=${property.service}" + } else { + "${property.authUrl}?$SERVICE=${property.service}" + } + property.scope?.let { + result += "&$SCOPE=${property.scope}" + } + userName?.let { result += "&account=$userName" } + return result + } +} \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt index da5a5b6217..803642cd94 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt @@ -31,10 +31,16 @@ package com.tencent.bkrepo.common.artifact.util.http +import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.CharPool.QUESTION import com.tencent.bkrepo.common.api.constant.CharPool.SLASH +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.constant.StringPool.HTTP import com.tencent.bkrepo.common.api.constant.StringPool.HTTPS +import com.tencent.bkrepo.common.storage.innercos.retry +import java.net.HttpURLConnection +import java.net.MalformedURLException +import java.net.URL /** * Http URL 格式化工具类 @@ -82,4 +88,81 @@ object UrlFormatter { } return url } + + + /** + * 拼接url + */ + fun buildUrl( + url: String, + path: String = StringPool.EMPTY, + params: String = StringPool.EMPTY, + ): String { + if (url.isBlank()) + throw IllegalArgumentException("Url should not be blank") + val newUrl = addProtocol(url.trim().trimEnd(SLASH)) + val baseUrl = URL(newUrl, newUrl.path) + val baseParams = newUrl.query + val builder = StringBuilder(baseUrl.toString().trimEnd(SLASH)) + if (path.isNotBlank()) { + builder.append(SLASH).append(path.trimStart(SLASH)) + } + if (!baseParams.isNullOrEmpty()) { + builder.append(QUESTION).append(baseParams) + } + if (params.isNotBlank()) { + if (builder.contains(QUESTION)) { + builder.append(CharPool.AND).append(params) + } else { + builder.append(QUESTION).append(params) + } + } + return builder.toString() + } + + /** + * 当没有protocol时进行添加 + */ + fun addProtocol(registry: String): URL { + try { + return URL(registry) + } catch (ignore: MalformedURLException) { + } + return addProtocolToHost(registry) + } + + /** + * 针对url如果没传protocol， 则默认以https请求发送； + * 如果http请求无法访问，则以http发送 + */ + private fun addProtocolToHost(registry: String): URL { + val url = try { + URL("$HTTPS$registry") + } catch (ignore: MalformedURLException) { + throw IllegalArgumentException("Check your input url!") + } + return try { + retry(times = 3, delayInSeconds = 1) { + validateHttpsProtocol(url) + url + } + } catch (ignore: Exception) { + URL(url.toString().replaceFirst("^https".toRegex(), "http")) + } + } + + /** + * 验证registry是否支持https + */ + private fun validateHttpsProtocol(url: URL): Boolean { + return try { + val http: HttpURLConnection = url.openConnection() as HttpURLConnection + http.instanceFollowRedirects = false + http.responseCode + http.disconnect() + true + } catch (e: Exception) { + throw e + } + } } diff --git a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt new file mode 100644 index 0000000000..582bc435b5 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt @@ -0,0 +1,176 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.util.http + +import com.tencent.bkrepo.common.api.constant.StringPool +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertAll + +internal class UrlFormatterTest { + + @Test + fun buildUrl() { + val url = "" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + try { + val result = UrlFormatter.buildUrl(url, path, params) + } catch (e: Exception) { + assertTrue(e.message!!.contains("Url should not be blank")) + } + + } + + @Test + fun buildUrl1() { + val url = "bkrepo.example.com" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl2() { + val url = "http://bkrepo.example.com" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl3() { + val url = "http://bkrepo.example.com/" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl4() { + val url = "http://bkrepo.example.com//" + val path = "/v2/" + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2/", result)} + ) + } + + @Test + fun buildUrl5() { + val url = "http://bkrepo.example.com//" + val path = "v2" + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2", result)} + ) + } + + @Test + fun buildUrl6() { + val url = "http://bkrepo.example.com//" + val path = "v2" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2?a=a", result)} + ) + } + + @Test + fun buildUrl7() { + val url = "http://bkrepo.example.com/?b=b" + val path = "v2" + val params = "a=a" + + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2?b=b&a=a", result)} + ) + } + + @Test + fun buildUrl8() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2" + + val result = UrlFormatter.buildUrl(url, path) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2", result)} + ) + } + + @Test + fun addProtocol() { + val url = "bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol1() { + val url = "http://bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol2() { + val url = "https://bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("https://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol3() { + val url = "bkrepo.example.com:test/" + try { + val result = UrlFormatter.addProtocol(url).toString() + } catch (e: Exception) { + assertTrue(e.message!!.contains("Check your input url!")) + } + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt index e4ba11d578..8060476568 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.batch import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.helm.api.HelmClient @@ -38,6 +39,7 @@ import com.tencent.bkrepo.job.batch.base.DefaultRepoJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.RepoInitJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -51,7 +53,8 @@ import java.time.LocalDateTime @EnableConfigurationProperties(RepoInitJobProperties::class) class RepoInitJob( private val properties: RepoInitJobProperties, - private val helmClient: HelmClient + private val helmClient: HelmClient, + private val ociClient: OciClient ) : DefaultRepoJob(properties) { private val types: List @@ -78,8 +81,16 @@ class RepoInitJob( try { val config = configuration.readJsonString() if (checkConfigType(config)) { - logger.info("Init request will be sent in repo $projectId|$name") - helmClient.initIndexAndPackage(projectId, name) + logger.info("init request will be sent in repo $projectId|$name") + when (row.type) { + RepositoryType.HELM.name -> { + helmClient.initIndexAndPackage(projectId, name) + } + RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { + ociClient.pullThirdPartyPackages(projectId, name) + } + else -> throw UnsupportedOperationException() + } } } catch (e: Exception) { throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt index 23a1776c85..40d7c76d16 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.batch import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.helm.api.HelmClient @@ -37,6 +38,7 @@ import com.tencent.bkrepo.job.batch.base.DefaultRepoJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.RepoRefreshJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -49,7 +51,8 @@ import org.springframework.stereotype.Component @EnableConfigurationProperties(RepoRefreshJobProperties::class) class RepoRefreshJob( private val properties: RepoRefreshJobProperties, - private val helmClient: HelmClient + private val helmClient: HelmClient, + private val ociClient: OciClient ) : DefaultRepoJob(properties) { private val types: List @@ -75,7 +78,15 @@ class RepoRefreshJob( val config = configuration.readJsonString() if (checkConfigType(config)) { logger.info("Refresh request will be sent in repo $projectId|$name") - helmClient.refreshIndexYamlAndPackage(projectId, name) + when (row.type) { + RepositoryType.HELM.name -> { + helmClient.refreshIndexYamlAndPackage(projectId, name) + } + RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { + ociClient.pullThirdPartyPackages(projectId, name) + } + else -> throw UnsupportedOperationException() + } } } catch (e: Exception) { throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt index 77fa5df3a4..13c006d39c 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt @@ -66,6 +66,7 @@ abstract class DefaultRepoJob( data class ProxyRepoData(private val map: Map) { val name: String by map val projectId: String by map + val type: String by map val configuration: String by map } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt index 778f07edca..386f585b21 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt @@ -36,5 +36,5 @@ open class RepoJobProperties( /** * 需要特殊处理的仓库类型 * */ - var repositorytypes: List = listOf("HELM") + var repositorytypes: List = listOf("HELM", "OCI", "DOCKER") ) : MongodbJobProperties() diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt index 380d3027dd..5f422574af 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt @@ -34,12 +34,12 @@ import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import org.springframework.cloud.openfeign.FeignClient import org.springframework.context.annotation.Primary +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping - @Api("oci") @Primary @FeignClient(OCI_SERVICE_NAME, contextId = "OciClient") @@ -51,4 +51,11 @@ interface OciClient { fun packageCreate( @RequestBody record: OciReplicationRecordInfo ): Response + + @ApiOperation("定时从第三方仓库拉取对应的package信息") + @PostMapping("/pull/package/{projectId}/{repoName}") + fun pullThirdPartyPackages( + @PathVariable projectId: String, + @PathVariable repoName: String + ): Response } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index b990b70ea8..694832db1e 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -42,9 +42,6 @@ const val HTTP_FORWARDED_PROTO = "X-Forwarded-Proto" const val HTTP_PROTOCOL_HTTP = "http" const val HTTP_PROTOCOL_HTTPS = "https" const val HOST = "Host" -const val BEARER_REALM = "Bearer realm" -const val SERVICE = "service" -const val SCOPE = "scope" const val PATCH = "PATCH" const val POST = "POST" const val NODE_FULL_PATH = "fullPath" @@ -98,6 +95,9 @@ const val MD5 = "md5" const val OCI_MANIFEST = "manifest.json" const val STAGE_TAG = "stageTag" +const val REQUEST_TAG_LIST = "tagList" +const val REQUEST_IMAGE = "image" + // OCIScheme is the URL scheme for OCI-based requests const val OCI_SCHEME = "oci" @@ -118,6 +118,8 @@ const val LEGACY_CHART_LAYER_MEDIA_TYPE = "application/tar+gzip" const val OCI_IMAGE_MANIFEST_MEDIA_TYPE = "application/vnd.oci.image.manifest.v1+json" +const val DOCKER_DISTRIBUTION_MANIFEST_V2 = "application/vnd.docker.distribution.manifest.v2+json" + // Content Descriptor const val CONTENT_DESCRIPTOR_MEDIA_TYPE = "application/vnd.oci.descriptor.v1+json" diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt index 6e766bbb08..cc8226539b 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt @@ -36,7 +36,9 @@ enum class OciMessageCode(private val key: String) : MessageCode { OCI_REPO_NOT_FOUND("oci.repo.not.found"), OCI_DELETE_RULES("oci.delete.rules"), OCI_VERSION_NOT_FOUND("oci.version.not.found"), - OCI_MANIFEST_INVALID("oci.manifest.invalid") + OCI_MANIFEST_INVALID("oci.manifest.invalid"), + OCI_REMOTE_CONFIGURATION_ERROR("oci.remote.configuration.error"), + OCI_REMOTE_CREDENTIALS_INVALID("oci.remote.credentials.invalid") ; override fun getBusinessCode() = ordinal + 1 override fun getKey() = key diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt index 9d18bd98c6..e5658715d9 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt @@ -38,10 +38,6 @@ interface ImagePackageInfoPullExtension: ExtensionPoint { * 拉取第三方镜像仓库package信息,并存储到对应仓库中 */ fun queryAndCreateDockerPackageInfo( - projectId: String, - repoName:String, - remoteUrl: String, - userName: String? = null, - password: String? = null + context: ImagePackagePullContext ) } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt new file mode 100644 index 0000000000..72d0ba4b88 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.extension + +import java.net.URL + +data class ImagePackagePullContext( + val projectId: String, + val repoName:String, + val remoteUrl: URL, + val userName: String? = null, + val password: String? = null +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt new file mode 100644 index 0000000000..c6c9363060 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.pojo.remote + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.oci.constant.REQUEST_IMAGE + +data class RemoteRequestProperty( + var url: String, + var imageName: String, + var fullPath: String = StringPool.EMPTY, + var params: String = StringPool.EMPTY, + var type: String = REQUEST_IMAGE +) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index e44181617b..b89a7ba6c4 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -31,13 +31,19 @@ package com.tencent.bkrepo.oci.artifact.repository +import com.google.common.cache.CacheBuilder import com.tencent.bkrepo.common.api.constant.BEARER_AUTH_PREFIX +import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpHeaders.WWW_AUTHENTICATE import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.util.AuthenticationUtil +import com.tencent.bkrepo.common.api.util.BasicAuthUtils import com.tencent.bkrepo.common.api.util.JsonUtils +import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext @@ -50,16 +56,17 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.artifactStream import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter -import com.tencent.bkrepo.oci.constant.BEARER_REALM +import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_V2 import com.tencent.bkrepo.oci.constant.DOCKER_LINK import com.tencent.bkrepo.oci.constant.LAST_TAG import com.tencent.bkrepo.oci.constant.MEDIA_TYPE import com.tencent.bkrepo.oci.constant.N import com.tencent.bkrepo.oci.constant.OCI_API_PREFIX +import com.tencent.bkrepo.oci.constant.OCI_FILTER_ENDPOINT import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE import com.tencent.bkrepo.oci.constant.OciMessageCode -import com.tencent.bkrepo.oci.constant.SCOPE -import com.tencent.bkrepo.oci.constant.SERVICE +import com.tencent.bkrepo.oci.constant.PROXY_URL +import com.tencent.bkrepo.oci.constant.REQUEST_TAG_LIST import com.tencent.bkrepo.oci.exception.OciForbiddenRequestException import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX @@ -68,13 +75,14 @@ import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.auth.BearerToken import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.remote.RemoteRequestProperty import com.tencent.bkrepo.oci.pojo.response.CatalogResponse +import com.tencent.bkrepo.oci.pojo.response.OciResponse import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import okhttp3.Credentials import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response @@ -83,6 +91,7 @@ import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.io.InputStream import java.net.URL +import java.util.concurrent.TimeUnit import java.util.regex.Pattern import javax.ws.rs.core.UriBuilder @@ -91,9 +100,15 @@ class OciRegistryRemoteRepository( private val ociOperationService: OciOperationService ) : RemoteRepository() { + private val clientCache = CacheBuilder.newBuilder().maximumSize(100) + .expireAfterWrite(60, TimeUnit.MINUTES).build() + private val tokenCache = CacheBuilder.newBuilder().maximumSize(100) + .expireAfterWrite(60, TimeUnit.MINUTES).build() + + override fun upload(context: ArtifactUploadContext) { with(context) { - val message = "Forbidden to upload chart into a remote repository [$projectId/$repoName]" + val message = "Forbidden to upload artifact into a remote repository [$projectId/$repoName]" logger.warn(message) throw OciForbiddenRequestException(OciMessageCode.OCI_FILE_UPLOAD_FORBIDDEN, "$projectId|$repoName") } @@ -103,8 +118,13 @@ class OciRegistryRemoteRepository( * 下载制品 */ override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { - return getCacheArtifactResource(context) ?: run { - return doRequest(context) as ArtifactResource? + return if (context.artifactInfo is OciManifestArtifactInfo) { + // 同一镜像tag可能被覆盖更新，对于manifest.json文件每次都去远端拉取 + doRequest(context) as ArtifactResource? + } else { + getCacheArtifactResource(context) ?: run { + doRequest(context) as ArtifactResource? + } } } @@ -122,25 +142,31 @@ class OciRegistryRemoteRepository( */ private fun doRequest(context: ArtifactContext): Any? { val remoteConfiguration = context.getRemoteConfiguration() - val httpClient = createHttpClient(remoteConfiguration, false) - val downloadUrl = createRemoteDownloadUrl(context) + val httpClient = clientCache.getIfPresent(remoteConfiguration) ?: run { + clientCache.put(remoteConfiguration, createHttpClient(remoteConfiguration, false)) + clientCache.getIfPresent(remoteConfiguration) + } + val property = getRemoteRequestProperty(context) + val downloadUrl = createRemoteDownloadUrl(context, property) logger.info("Remote request $downloadUrl will be sent") - val request = buildRequest(downloadUrl, remoteConfiguration) - val response = httpClient.newCall(request).execute() + val tokenKey = buildTokenCacheKey(context.getStringAttribute(PROXY_URL)!!, property.imageName) + val request = buildRequest(downloadUrl, remoteConfiguration, tokenCache.getIfPresent(tokenKey)) + val response = httpClient!!.newCall(request).execute() var responseWithAuth: Response? = null try { if (response.isSuccessful) return onResponse(context, response) // 针对返回401进行token获取 - val token = getAuthenticationCode(response, remoteConfiguration, httpClient) + val token = getAuthenticationCode(context, response, remoteConfiguration, property.imageName) if (token.isNullOrBlank()) return null - val requestWithAuth = buildRequest( + tokenCache.put(tokenKey, token) + val requestWithToken = buildRequest( url = downloadUrl, configuration = remoteConfiguration, addBasicInterceptor = false, token = token ) responseWithAuth = httpClient - .newCall(requestWithAuth) + .newCall(requestWithToken) .execute() return if (checkResponse(responseWithAuth)) { onResponse(context, responseWithAuth) @@ -171,11 +197,15 @@ class OciRegistryRemoteRepository( addBasicInterceptor: Boolean = true ): Request { val requestBuilder = Request.Builder().url(url) - if (addBasicInterceptor) { + if (addBasicInterceptor && token.isNullOrEmpty()) { requestBuilder.addInterceptor(configuration) } else { token?.let { requestBuilder.header(HttpHeaders.AUTHORIZATION, token) } } + // 拉取第三方仓库时，默认会返回v1版本的镜像格式 + if (url.contains("/manifests/")) { + requestBuilder.header(HttpHeaders.ACCEPT, DOCKER_DISTRIBUTION_MANIFEST_V2) + } return requestBuilder.build() } @@ -183,7 +213,7 @@ class OciRegistryRemoteRepository( val username = configuration.credentials.username val password = configuration.credentials.password if (username != null && password != null) { - val credentials = Credentials.basic(username, password) + val credentials = BasicAuthUtils.encode(username, password) this.header(HttpHeaders.AUTHORIZATION, credentials) } return this @@ -193,61 +223,95 @@ class OciRegistryRemoteRepository( * 生成远程构件下载url */ override fun createRemoteDownloadUrl(context: ArtifactContext): String { - val configuration = context.getRemoteConfiguration() - if (context.artifactInfo is OciBlobArtifactInfo) { - val artifactInfo = context.artifactInfo as OciBlobArtifactInfo - return createUrl( - url = configuration.url, - fullPath = OciLocationUtils.blobPathLocation(artifactInfo.getDigest(), artifactInfo), - params = StringPool.EMPTY - ) - } - if (context.artifactInfo is OciManifestArtifactInfo) { - val artifactInfo = context.artifactInfo as OciManifestArtifactInfo - return createUrl( - url = configuration.url, - fullPath = OciLocationUtils.manifestPathLocation(artifactInfo.reference, artifactInfo), - params = StringPool.EMPTY - ) + val property = getRemoteRequestProperty(context) + return createRemoteDownloadUrl(context, property) + } + + fun createRemoteDownloadUrl(context: ArtifactContext, property: RemoteRequestProperty): String { + return when (property.type) { + REQUEST_TAG_LIST -> createCatalogUrl(property) + else -> createUrl(property) } - if (context.artifactInfo is OciTagArtifactInfo) { - val artifactInfo = context.artifactInfo as OciTagArtifactInfo - if (artifactInfo.packageName.isBlank()) { - val (_, params) = createParamsForTagList(context) - return createCatalogUrl(configuration.url, params) - } else { - val (fullPath, params) = createParamsForTagList(context) - return createUrl(configuration.url, fullPath, params) + } + + /** + * 获取不同情况下对应的请求属性 + */ + private fun getRemoteRequestProperty(context: ArtifactContext): RemoteRequestProperty { + val configuration = context.getRemoteConfiguration() + val url = UrlFormatter.addProtocol(configuration.url).toString() + context.putAttribute(PROXY_URL, url) + return when (context.artifactInfo) { + is OciBlobArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciBlobArtifactInfo + RemoteRequestProperty( + url = url, + fullPath = OciLocationUtils.blobPathLocation(artifactInfo.getDigest(), artifactInfo), + imageName = artifactInfo.packageName + ) + } + is OciTagArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciTagArtifactInfo + if (artifactInfo.packageName.isBlank()) { + val (_, params) = createParamsForTagList(context) + RemoteRequestProperty( + url = url, + params = params, + type = REQUEST_TAG_LIST, + imageName = StringPool.EMPTY + ) + } else { + val (fullPath, params) = createParamsForTagList(context) + RemoteRequestProperty( + url = url, + fullPath = fullPath, + params = params, + imageName = artifactInfo.packageName + ) + } } + is OciManifestArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciManifestArtifactInfo + RemoteRequestProperty( + url = url, + fullPath = OciLocationUtils.manifestPathLocation(artifactInfo.reference, artifactInfo), + imageName = artifactInfo.packageName + ) + } + else -> RemoteRequestProperty(url = url, imageName = StringPool.EMPTY) } - return createUrl(configuration.url) } /** * 拼接url */ - private fun createUrl(url: String, fullPath: String = StringPool.EMPTY, params: String = StringPool.EMPTY): String { - val baseUrl = URL(url) - val v2Url = URL(baseUrl, "/v2" + baseUrl.path) - return UrlFormatter.format(v2Url.toString(), fullPath, params) + private fun createUrl(property: RemoteRequestProperty): String { + with(property) { + val baseUrl = URL(url) + val v2Url = URL(baseUrl, OCI_FILTER_ENDPOINT + baseUrl.path) + return UrlFormatter.format(v2Url.toString(), fullPath, params) + } } /** * 拼接catalog url */ - private fun createCatalogUrl(url: String, params: String = StringPool.EMPTY): String { - val baseUrl = URL(url) - val builder = UriBuilder.fromPath(OCI_API_PREFIX) - .host(baseUrl.host).scheme(baseUrl.protocol) - .path(DOCKER_CATALOG_SUFFIX) - .queryParam(params) - return builder.build().toString() + private fun createCatalogUrl(property: RemoteRequestProperty): String { + with(property) { + val baseUrl = URL(url) + val builder = UriBuilder.fromPath(OCI_API_PREFIX) + .host(baseUrl.host).scheme(baseUrl.protocol) + .path(DOCKER_CATALOG_SUFFIX) + .queryParam(params) + return builder.build().toString() + } } private fun getAuthenticationCode( + context: ArtifactContext, response: Response, configuration: RemoteConfiguration, - httpClient: OkHttpClient + imageName: String ): String? { if (response.code != HttpStatus.UNAUTHORIZED.value) { return null @@ -256,47 +320,57 @@ class OciRegistryRemoteRepository( if (wwwAuthenticate.isNullOrBlank() || !wwwAuthenticate.startsWith(BEARER_AUTH_PREFIX)) { return null } - val url = parseWWWAuthenticateHeader(wwwAuthenticate) - logger.info("The url for authenticating is $url") - if (url.isNullOrEmpty()) return null + val url = context.getStringAttribute(PROXY_URL)!! + val scope = getScope(url, imageName) + val authProperty = AuthenticationUtil.parseWWWAuthenticateHeader(wwwAuthenticate, scope) + if (authProperty == null) { + logger.warn("Auth url can not be parsed from header!") + return null + } + val urlStr = AuthenticationUtil.buildAuthenticationUrl(authProperty, configuration.credentials.username) + logger.info("The url for authenticating is $urlStr") + if (urlStr.isNullOrEmpty()) return null val request = buildRequest( - url = url, + url = urlStr, configuration = configuration, - addBasicInterceptor = false + addBasicInterceptor = true ) - val tokenResponse = httpClient.newCall(request).execute() - try { - if (!tokenResponse.isSuccessful) return null - val body = tokenResponse.body!! - val artifactFile = createTempFile(body) - val size = artifactFile.getSize() - val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size)) - artifactFile.delete() - val bearerToken = JsonUtils.objectMapper.readValue(artifactStream, BearerToken::class.java) - return "Bearer ${bearerToken.token}" - } finally { - tokenResponse.body?.close() + clientCache.getIfPresent(configuration)!!.newCall(request).execute().use { + if (!it.isSuccessful) { + val error = try { + JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java).toJsonString() + } catch (ignore: Exception) { + StringPool.EMPTY + } + val errMsg = "Could not get token from auth service," + + " code is ${it.code} and response is $error" + logger.warn(errMsg) + throw ErrorCodeException( + OciMessageCode.OCI_REMOTE_CONFIGURATION_ERROR, + errMsg + ) + } + try { + val bearerToken = JsonUtils.objectMapper.readValue(it.body!!.byteStream(), BearerToken::class.java) + return "Bearer ${bearerToken.token}" + } catch (e: Exception) { + throw ErrorCodeException( + OciMessageCode.OCI_REMOTE_CONFIGURATION_ERROR, + "Could not get token from auth service, please check your remote configuration!" + ) + } } + } /** - * 解析返回头中的WWW_AUTHENTICATE字段， 只针对为Bearer realm + * 可能请求的地址一样，但是 */ - private fun parseWWWAuthenticateHeader(wwwAuthenticate: String): String? { - val map: MutableMap = mutableMapOf() - return try { - val params = wwwAuthenticate.split(",") - params.forEach { - val param = it.split("=") - val name = param.first() - val value = param.last().removeSurrounding("\"") - map[name] = value - } - "${map[BEARER_REALM]}?$SERVICE=${map[SERVICE]}&$SCOPE=${map[SCOPE]}" - } catch (e: Exception) { - logger.warn("Parsing wwwAuthenticate header error: ${e.message}") - null - } + private fun getScope(remoteUrl: String, imageName: String): String { + val baseUrl = URL(remoteUrl) + val target = baseUrl.path.removePrefix(StringPool.SLASH) + .removeSuffix(StringPool.SLASH) + StringPool.SLASH + imageName + return "repository:$target:pull" } private fun createParamsForTagList(context: ArtifactContext): Pair { @@ -412,13 +486,14 @@ class OciRegistryRemoteRepository( // 针对manifest文件获取会通过tag或manifest获取，避免重复创建 fullPath?.let { val node = nodeClient.getNodeDetail(ociArtifactInfo.projectId, ociArtifactInfo.repoName, fullPath).data - if (node != null) return node + if (node != null && artifactFile.getFileSha256() == node.sha256) return node } + val url = context.getStringAttribute(PROXY_URL) var nodeDetail = ociOperationService.storeArtifact( ociArtifactInfo = ociArtifactInfo, artifactFile = artifactFile, storageCredentials = context.storageCredentials, - proxyUrl = configuration.url + proxyUrl = url ) // 针对manifest文件需要更新metadata if (context.artifactInfo is OciManifestArtifactInfo) { @@ -435,7 +510,7 @@ class OciRegistryRemoteRepository( private fun updateManifestAndBlob(context: ArtifactDownloadContext, nodeDetail: NodeDetail) { with(context.artifactInfo as OciManifestArtifactInfo) { val digest = OciDigest.fromSha256(nodeDetail.sha256!!) - // 上传manifest文件，同时需要将manifest中对应blob的属性进行补充到blob节点中，同时创建package相关信息 + // 上传manifest文件，同时创建package相关信息 ociOperationService.updateOciInfo( ociArtifactInfo = this, digest = digest, @@ -506,6 +581,11 @@ class OciRegistryRemoteRepository( return n } + private fun buildTokenCacheKey(remoteUrl: String, imageName: String): String { + val scope = getScope(remoteUrl, imageName) + return "$remoteUrl${CharPool.COLON}$scope" + } + companion object { val logger: Logger = LoggerFactory.getLogger(OciRegistryRemoteRepository::class.java) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt index aa7bbe8695..35aa01fb91 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt @@ -42,8 +42,8 @@ import org.springframework.web.bind.annotation.RestController @RestController class OciPackageController( private val operationService: OciOperationService, - private val ociReplicationRecordDao: OciReplicationRecordDao -): OciClient { + private val ociReplicationRecordDao: OciReplicationRecordDao, + ): OciClient { override fun packageCreate(record: OciReplicationRecordInfo): Response { with(record) { val ociArtifactInfo = OciManifestArtifactInfo( @@ -65,4 +65,9 @@ class OciPackageController( return ResponseBuilder.success() } } + + override fun pullThirdPartyPackages(projectId: String, repoName: String): Response { + operationService.getPackagesFromThirdPartyRepo(projectId, repoName) + return ResponseBuilder.success() + } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt index e743922882..dd4ca9e21f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.oci.listener.base import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel @@ -57,7 +58,7 @@ open class EventExecutor( ): Future { return threadPoolExecutor.submit { try { - replicationEventHandler(event) + eventHandler(event) true } catch (exception: Throwable) { logger.warn("Error occurred while executing the oci event: $exception") @@ -66,6 +67,16 @@ open class EventExecutor( } } + private fun eventHandler(event: ArtifactEvent) { + when (event.type) { + EventType.REPLICATION_THIRD_PARTY -> replicationEventHandler(event) + EventType.REPO_CREATED, EventType.REPO_REFRESHED, EventType.REPO_UPDATED -> { + ociOperationService.getPackagesFromThirdPartyRepo(event.projectId, event.repoName) + } + else -> UnsupportedOperationException() + } + } + private fun replicationEventHandler(event: ArtifactEvent) { with(event) { val packageName = event.data["packageName"].toString() diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt new file mode 100644 index 0000000000..6e124377d9 --- /dev/null +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.listener.consumer + +import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType +import com.tencent.bkrepo.oci.listener.base.EventExecutor +import com.tencent.bkrepo.oci.service.OciOperationService +import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.repository.api.RepositoryClient +import org.slf4j.LoggerFactory +import org.springframework.messaging.Message +import org.springframework.stereotype.Component +import java.util.function.Consumer + +/** + * 构件事件消费者，用于实时同步 + * 对应destination为对应ArtifactEvent.topic + */ +@Component("remoteOciRepo") +class RemoteImageRepoEventConsumer( + override val nodeClient: NodeClient, + override val repositoryClient: RepositoryClient, + override val ociOperationService: OciOperationService +) : Consumer> +, EventExecutor(nodeClient, repositoryClient, ociOperationService){ + + /** + * 允许接收的事件类型 + */ + private val acceptTypes = setOf( + EventType.REPO_CREATED, + EventType.REPO_UPDATED, + EventType.REPO_REFRESHED + ) + + override fun accept(message: Message) { + if (!acceptTypes.contains(message.payload.type)) { + return + } + logger.info("current repo operation message header is ${message.headers}") + submit(message.payload) + } + + companion object { + private val logger = LoggerFactory.getLogger(RemoteImageRepoEventConsumer::class.java) + } +} diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index ad696b23e6..3cb99c425d 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -193,4 +193,9 @@ interface OciOperationService { pageSize: Int, tag: String? ): OciTagResult + + /** + * 拉取第三方镜像仓库package信息 + */ + fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 407a223a62..b00361894a 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -32,14 +32,19 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE +import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration +import com.tencent.bkrepo.common.artifact.pojo.configuration.composite.CompositeConfiguration +import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils @@ -68,6 +73,8 @@ import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.exception.OciFileNotFoundException import com.tencent.bkrepo.oci.exception.OciVersionNotFoundException +import com.tencent.bkrepo.oci.extension.ImagePackageInfoPullExtension +import com.tencent.bkrepo.oci.extension.ImagePackagePullContext import com.tencent.bkrepo.oci.model.Descriptor import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.TOciReplicationRecord @@ -99,6 +106,8 @@ import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder import com.tencent.bkrepo.repository.pojo.search.PackageQueryBuilder +import com.tencent.devops.plugin.api.PluginManager +import com.tencent.devops.plugin.api.applyExtension import org.apache.commons.lang3.StringUtils import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -119,8 +128,9 @@ class OciOperationServiceImpl( private val storageManager: StorageManager, private val repositoryClient: RepositoryClient, private val ociProperties: OciProperties, - private val ociReplicationRecordDao: OciReplicationRecordDao -) : OciOperationService { + private val ociReplicationRecordDao: OciReplicationRecordDao, + private val pluginManager: PluginManager + ) : OciOperationService { /** * 检查package 对应的version是否存在 @@ -1093,6 +1103,47 @@ class OciOperationServiceImpl( return OciTagResult(result.totalRecords, data) } + override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String,) { + val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") + buildImagePackagePullContext(projectId, repoName, repositoryDetail.configuration).forEach { + pluginManager.applyExtension { + queryAndCreateDockerPackageInfo(it) + } + } + } + + private fun buildImagePackagePullContext( + projectId: String, + repoName: String, + config: RepositoryConfiguration + ): List { + return when (config) { + is RemoteConfiguration -> { + listOf( + ImagePackagePullContext( + projectId = projectId, + repoName = repoName, + remoteUrl = UrlFormatter.addProtocol(config.url), + userName = config.credentials.username, + password = config.credentials.password + ) + ) + } + is CompositeConfiguration -> { + config.proxy.channelList.map { + ImagePackagePullContext( + projectId = projectId, + repoName = repoName, + remoteUrl = UrlFormatter.addProtocol(it.url), + userName = it.username, + password = it.password + ) + } + } + else -> throw UnsupportedOperationException() + } + } + private fun buildString(stageTag: List): String { if (stageTag.isEmpty()) return StringPool.EMPTY return StringUtils.join(stageTag.toTypedArray()).toString() diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties index d7d60fc7dc..0749d8bb5a 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties @@ -35,4 +35,6 @@ oci.file.upload.forbidden=Artifact is forbidden to upload in repo [{0}]! oci.repo.not.found=Oci repository [{0}] not found! oci.delete.rules=Delete the blob [{0}] identified by digest! oci.version.not.found=The version [{0}] of artifact not found in repo [{1}] ! -oci.manifest.invalid=Manifest invalid! \ No newline at end of file +oci.manifest.invalid=Manifest invalid! +oci.remote.configuration.error= Remote configuration error, error is [{0}]! +oci.remote.credentials.invalid= Remote credentials invalid, error is [{0}]! \ No newline at end of file diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties index a96b6e117a..14f13c387c 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties @@ -35,4 +35,6 @@ oci.file.upload.forbidden=仓库 [{0}] 禁止上传制品！ oci.repo.not.found=Oci仓库 [{0}] 不存在！ oci.delete.rules=制品 [{0}] 只能使用digest删除！ oci.version.not.found=制品版本 [{0}] 在仓库 [{1}] 中不存在！ -oci.manifest.invalid=请检查Manifest内容是否符合规范！ \ No newline at end of file +oci.manifest.invalid=请检查Manifest内容是否符合规范！ +oci.remote.configuration.error= 请检查远程仓库配置信息, 错误：[{0}]! +oci.remote.credentials.invalid= 代理鉴权信息有误: [{0}]，请确认！ \ No newline at end of file diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties index 0b9bbb78f0..80014534de 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties @@ -35,4 +35,6 @@ oci.file.upload.forbidden=倉庫 [{0}] 禁止上傳製品！ oci.repo.not.found=Oci倉庫 [{0}] 不存在！ oci.delete.rules=製品 [{0}] 只能digest刪除！ oci.version.not.found=製品版本 [{0}] 在倉庫 [{1}] 中不存在！ -oci.manifest.invalid=請檢查Manifest內容是否符合規範！ \ No newline at end of file +oci.manifest.invalid=請檢查Manifest內容是否符合規範！ +oci.remote.configuration.error= 請檢查遠程倉庫配置信息, 錯誤：[{0}]! +oci.remote.credentials.invalid= 代理鑒權信息有誤: [{0}] ，請確認！ \ No newline at end of file diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt index f98f68e889..0c2e8b5be0 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt @@ -29,9 +29,6 @@ package com.tencent.bkrepo.replication.constant const val DEFAULT_VERSION = "1.0.0" -const val BEARER_REALM = "Bearer realm" -const val SERVICE = "service" -const val SCOPE = "scope" const val REPOSITORY = "repository" const val URL = "url" diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt index dd029ce030..ee395dce11 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.constant.CHUNKED_UPLOAD import com.tencent.bkrepo.replication.constant.REPOSITORY_INFO @@ -44,7 +45,6 @@ import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.replica.base.context.FilePushContext import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.util.HttpUtils import com.tencent.bkrepo.replication.util.StreamRequestBody import okhttp3.Headers import okhttp3.MediaType.Companion.toMediaTypeOrNull @@ -352,7 +352,7 @@ abstract class ArtifactReplicationHandler( } catch (e: Exception) { val baseUrl = URL(url) val host = URL(baseUrl.protocol, baseUrl.host, StringPool.EMPTY).toString() - HttpUtils.buildUrl(host, location.removePrefix("/")) + UrlFormatter.buildUrl(host, location.removePrefix("/")) } } } @@ -368,7 +368,7 @@ abstract class ArtifactReplicationHandler( ): String { val baseUrl = URL(url) val suffixUrl = URL(baseUrl, baseUrl.path).toString() - return HttpUtils.buildUrl(suffixUrl, path, params) + return UrlFormatter.buildUrl(suffixUrl, path, params) } open fun buildRequestTag( diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt index 2a7ca766f8..989ac50cf1 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt @@ -31,14 +31,11 @@ import com.tencent.bkrepo.common.api.constant.BEARER_AUTH_PREFIX import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.StringPool -import com.tencent.bkrepo.common.api.constant.StringPool.QUESTION +import com.tencent.bkrepo.common.api.util.AuthenticationUtil.buildAuthenticationUrl +import com.tencent.bkrepo.common.api.util.AuthenticationUtil.parseWWWAuthenticateHeader import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.toJsonString -import com.tencent.bkrepo.replication.constant.BEARER_REALM -import com.tencent.bkrepo.replication.constant.SCOPE -import com.tencent.bkrepo.replication.constant.SERVICE import com.tencent.bkrepo.replication.pojo.docker.OciResponse -import com.tencent.bkrepo.replication.pojo.remote.AuthenticationProperty import com.tencent.bkrepo.replication.pojo.remote.BearerToken import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.replica.base.impl.remote.base.AuthorizationService @@ -106,10 +103,14 @@ class OciAuthorizationService : AuthorizationService { val httpRequest = HttpUtils.wrapperRequest(property) httpClient.newCall(httpRequest).execute().use { if (!it.isSuccessful) { - val error = JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java) + val error = try { + JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java).toJsonString() + } catch (ignore: Exception) { + StringPool.EMPTY + } throw ArtifactPushException( "Could not get token from auth service," + - " code is ${it.code} and response is ${error.toJsonString()}" + " code is ${it.code} and response is $error" ) } try { @@ -123,44 +124,6 @@ class OciAuthorizationService : AuthorizationService { } } - /** - * 解析返回头中的WWW_AUTHENTICATE字段， 只针对为Bearer realm - */ - private fun parseWWWAuthenticateHeader(wwwAuthenticate: String, scope: String?): AuthenticationProperty? { - val map: MutableMap = mutableMapOf() - return try { - val params = wwwAuthenticate.split("\",") - params.forEach { - val param = it.split(Regex("="),2) - val name = param.first() - val value = param.last().replace("\"", "") - map[name] = value - } - AuthenticationProperty( - authUrl = map[BEARER_REALM]!!, - service = map[SERVICE]!!, - scope = scope - ) - } catch (e: Exception) { - logger.warn("Parsing wwwAuthenticate header error: ${e.message}") - null - } - } - - private fun buildAuthenticationUrl(property: AuthenticationProperty, userName: String?): String? { - if (property.authUrl.isBlank()) return null - var result = if (property.authUrl.contains(QUESTION)) { - "${property.authUrl}&$SERVICE=${property.service}" - } else { - "${property.authUrl}?$SERVICE=${property.service}" - } - property.scope?.let { - result += "&$SCOPE=${property.scope}" - } - userName?.let { result += "&account=$userName" } - return result - } - companion object { private val logger = LoggerFactory.getLogger(OciAuthorizationService::class.java) } diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt index 082ad9fdbb..4cb6ffba58 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.artifact.event.packages.VersionCreatedEvent import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter.addProtocol import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace import com.tencent.bkrepo.replication.api.ReplicaTaskOperationClient @@ -69,7 +70,6 @@ import com.tencent.bkrepo.replication.service.RemoteNodeService import com.tencent.bkrepo.replication.service.ReplicaNodeDispatchService import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.replication.service.ReplicaTaskService -import com.tencent.bkrepo.replication.util.HttpUtils.addProtocol import com.tencent.bkrepo.replication.util.ReplicationMetricsRecordUtil.convertToReplicationTaskMetricsRecord import com.tencent.bkrepo.replication.util.ReplicationMetricsRecordUtil.toJson import org.slf4j.LoggerFactory diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt index 914f02e927..b63779d48a 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt @@ -27,19 +27,14 @@ package com.tencent.bkrepo.replication.util -import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.HttpHeaders -import com.tencent.bkrepo.common.api.constant.StringPool -import com.tencent.bkrepo.common.storage.innercos.retry -import com.tencent.bkrepo.replication.constant.DELAY_IN_SECONDS -import com.tencent.bkrepo.replication.constant.RETRY_COUNT +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter.buildUrl import com.tencent.bkrepo.replication.pojo.blob.RequestTag import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import okhttp3.Request import org.springframework.web.bind.annotation.RequestMethod import java.io.IOException import java.net.HttpURLConnection -import java.net.MalformedURLException import java.net.URL object HttpUtils { @@ -73,46 +68,6 @@ object HttpUtils { } } - /** - * 拼接url - */ - fun buildUrl( - url: String, - path: String = StringPool.EMPTY, - params: String = StringPool.EMPTY, - ): String { - val builder = StringBuilder(url) - if (path.isNotBlank()) { - builder.append(CharPool.SLASH).append(path.trimStart(CharPool.SLASH)) - } - if (params.isNotBlank()) { - if (builder.contains(CharPool.QUESTION)) { - builder.append(CharPool.AND).append(params) - } else { - builder.append(CharPool.QUESTION).append(params) - } - } - return builder.toString() - } - - /** - * 针对url如果没传protocol， 则默认以https请求发送 - */ - fun addProtocol(registry: String): URL { - try { - return URL(registry) - } catch (ignore: MalformedURLException) { - } - val url = URL("${StringPool.HTTPS}$registry") - return try { - retry(times = RETRY_COUNT, delayInSeconds = DELAY_IN_SECONDS) { - validateHttpsProtocol(url) - url - } - } catch (ignore: Exception) { - URL(url.toString().replaceFirst("^https".toRegex(), "http")) - } - } /** * Pings a HTTP URL. This effectively sends a HEAD request and returns `true` if the response code is in @@ -139,21 +94,6 @@ object HttpUtils { } } - /** - * 验证registry是否支持https - */ - private fun validateHttpsProtocol(url: URL): Boolean { - return try { - val http: HttpURLConnection = url.openConnection() as HttpURLConnection - http.instanceFollowRedirects = false - http.responseCode - http.disconnect() - true - } catch (e: Exception) { - throw e - } - } - /** * 从Content-Range头中解析出起始位置 */ From 6bec895a3e136c5123862a1855e238661f7364f6 Mon Sep 17 00:00:00 2001 From: zacYL Date: Mon, 3 Jul 2023 19:53:18 +0800 Subject: [PATCH 03/41] =?UTF-8?q?bug:=20=E6=8E=A5=E5=8F=A3=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/job/batch/RepoInitJob.kt | 2 +- .../com/tencent/bkrepo/job/batch/RepoRefreshJob.kt | 2 +- .../kotlin/com/tencent/bkrepo/oci/api/OciClient.kt | 2 +- .../oci/controller/service/OciPackageController.kt | 12 ++++++++++-- .../bkrepo/oci/listener/base/EventExecutor.kt | 4 +++- .../consumer/RemoteImageRepoEventConsumer.kt | 12 +++--------- .../listener/consumer/ReplicationEventListener.kt | 11 +++-------- .../consumer/ThirdPartyReplicationEventConsumer.kt | 13 +++---------- 8 files changed, 25 insertions(+), 33 deletions(-) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt index 8060476568..310bb3b81a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt @@ -87,7 +87,7 @@ class RepoInitJob( helmClient.initIndexAndPackage(projectId, name) } RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { - ociClient.pullThirdPartyPackages(projectId, name) + ociClient.getPackagesFromThirdPartyRepo(projectId, name) } else -> throw UnsupportedOperationException() } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt index 40d7c76d16..bbccf869a7 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt @@ -83,7 +83,7 @@ class RepoRefreshJob( helmClient.refreshIndexYamlAndPackage(projectId, name) } RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { - ociClient.pullThirdPartyPackages(projectId, name) + ociClient.getPackagesFromThirdPartyRepo(projectId, name) } else -> throw UnsupportedOperationException() } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt index 5f422574af..d3251d105b 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt @@ -54,7 +54,7 @@ interface OciClient { @ApiOperation("定时从第三方仓库拉取对应的package信息") @PostMapping("/pull/package/{projectId}/{repoName}") - fun pullThirdPartyPackages( + fun getPackagesFromThirdPartyRepo( @PathVariable projectId: String, @PathVariable repoName: String ): Response diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt index 35aa01fb91..7cf57840f1 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt @@ -28,9 +28,12 @@ package com.tencent.bkrepo.oci.controller.service import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.event.repo.RepoCreatedEvent +import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.oci.api.OciClient import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao +import com.tencent.bkrepo.oci.listener.base.EventExecutor import com.tencent.bkrepo.oci.model.TOciReplicationRecord import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.third.OciReplicationRecordInfo @@ -43,6 +46,7 @@ import org.springframework.web.bind.annotation.RestController class OciPackageController( private val operationService: OciOperationService, private val ociReplicationRecordDao: OciReplicationRecordDao, + private val eventExecutor: EventExecutor ): OciClient { override fun packageCreate(record: OciReplicationRecordInfo): Response { with(record) { @@ -66,8 +70,12 @@ class OciPackageController( } } - override fun pullThirdPartyPackages(projectId: String, repoName: String): Response { - operationService.getPackagesFromThirdPartyRepo(projectId, repoName) + override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String): Response { + eventExecutor.submit(RepoCreatedEvent( + projectId = projectId, + repoName = repoName, + userId = SecurityUtils.getUserId() + )) return ResponseBuilder.success() } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt index dd4ca9e21f..0d822c99a8 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt @@ -40,10 +40,12 @@ import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.Logger import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component import java.util.concurrent.Future import java.util.concurrent.ThreadPoolExecutor -open class EventExecutor( +@Component +class EventExecutor( open val nodeClient: NodeClient, open val repositoryClient: RepositoryClient, open val ociOperationService: OciOperationService diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt index 6e124377d9..753b5fd69b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt @@ -30,9 +30,6 @@ package com.tencent.bkrepo.oci.listener.consumer import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.oci.listener.base.EventExecutor -import com.tencent.bkrepo.oci.service.OciOperationService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.messaging.Message import org.springframework.stereotype.Component @@ -44,11 +41,8 @@ import java.util.function.Consumer */ @Component("remoteOciRepo") class RemoteImageRepoEventConsumer( - override val nodeClient: NodeClient, - override val repositoryClient: RepositoryClient, - override val ociOperationService: OciOperationService -) : Consumer> -, EventExecutor(nodeClient, repositoryClient, ociOperationService){ + private val eventExecutor: EventExecutor +) : Consumer>{ /** * 允许接收的事件类型 @@ -64,7 +58,7 @@ class RemoteImageRepoEventConsumer( return } logger.info("current repo operation message header is ${message.headers}") - submit(message.payload) + eventExecutor.submit(message.payload) } companion object { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt index 5ae0f39e02..7a7e928732 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt @@ -29,9 +29,6 @@ package com.tencent.bkrepo.oci.listener.consumer import com.tencent.bkrepo.common.artifact.event.replication.ThirdPartyReplicationEvent import com.tencent.bkrepo.oci.listener.base.EventExecutor -import com.tencent.bkrepo.oci.service.OciOperationService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.springframework.context.event.EventListener import org.springframework.stereotype.Component @@ -40,15 +37,13 @@ import org.springframework.stereotype.Component */ @Component class ReplicationEventListener( - override val nodeClient: NodeClient, - override val repositoryClient: RepositoryClient, - override val ociOperationService: OciOperationService -): EventExecutor(nodeClient, repositoryClient, ociOperationService) { + private val eventExecutor: EventExecutor +){ /** * 第三方同步事件处理 */ @EventListener(ThirdPartyReplicationEvent::class) fun handle(event: ThirdPartyReplicationEvent) { - submit(event) + eventExecutor.submit(event) } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt index 9dad95d058..bad477b59f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt @@ -30,9 +30,6 @@ package com.tencent.bkrepo.oci.listener.consumer import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.oci.listener.base.EventExecutor -import com.tencent.bkrepo.oci.service.OciOperationService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.messaging.Message import org.springframework.stereotype.Component @@ -45,12 +42,8 @@ import java.util.function.Consumer */ @Component("thirdPartyReplication") class ThirdPartyReplicationEventConsumer( - override val nodeClient: NodeClient, - override val repositoryClient: RepositoryClient, - override val ociOperationService: OciOperationService -) : - Consumer>, - EventExecutor(nodeClient, repositoryClient, ociOperationService) { + private val eventExecutor: EventExecutor +) : Consumer> { /** * 允许接收的事件类型 @@ -64,7 +57,7 @@ class ThirdPartyReplicationEventConsumer( return } logger.info("current third party replication message header is ${message.headers}") - submit(message.payload) + eventExecutor.submit(message.payload) } companion object { From 974ad2478e7853498e319133afb1246c8a1e01eb Mon Sep 17 00:00:00 2001 From: zacYL Date: Mon, 3 Jul 2023 19:54:16 +0800 Subject: [PATCH 04/41] =?UTF-8?q?bug:=20=E5=BC=95=E7=94=A8=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../base/handler/RemoteClusterArtifactReplicationHandler.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt index 9b96f26361..8dec006a3b 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt @@ -28,13 +28,13 @@ package com.tencent.bkrepo.replication.replica.base.handler import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.constant.OCI_BLOBS_UPLOAD_FIRST_STEP_URL import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult import com.tencent.bkrepo.replication.replica.base.context.FilePushContext import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.util.HttpUtils import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.net.URL @@ -105,7 +105,7 @@ class RemoteClusterArtifactReplicationHandler( ): String { val baseUrl = URL(url) val suffixUrl = URL(baseUrl, "/v2" + baseUrl.path).toString() - return HttpUtils.buildUrl(suffixUrl, path, params) + return UrlFormatter.buildUrl(suffixUrl, path, params) } companion object { private val logger = LoggerFactory.getLogger(RemoteClusterArtifactReplicationHandler::class.java) From 0eb1d9dfd4556681e4d6e64c0109f753facb5425 Mon Sep 17 00:00:00 2001 From: zacYL Date: Mon, 3 Jul 2023 20:18:44 +0800 Subject: [PATCH 05/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=90=86url=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/OciOperationServiceImpl.kt | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index b00361894a..ae4c17cd8a 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -1117,31 +1117,41 @@ class OciOperationServiceImpl( repoName: String, config: RepositoryConfiguration ): List { - return when (config) { + val result = mutableListOf() + when (config) { is RemoteConfiguration -> { - listOf( - ImagePackagePullContext( + try { + val remoteUrl = UrlFormatter.addProtocol(config.url) + result.add(ImagePackagePullContext( projectId = projectId, repoName = repoName, - remoteUrl = UrlFormatter.addProtocol(config.url), + remoteUrl = remoteUrl, userName = config.credentials.username, password = config.credentials.password - ) - ) + )) + } catch (e: Exception) { + logger.warn("illegal remote url ${config.url} for repo $projectId|$repoName") + } } is CompositeConfiguration -> { - config.proxy.channelList.map { - ImagePackagePullContext( - projectId = projectId, - repoName = repoName, - remoteUrl = UrlFormatter.addProtocol(it.url), - userName = it.username, - password = it.password - ) + config.proxy.channelList.forEach { + try { + val remoteUrl = UrlFormatter.addProtocol(it.url) + result.add(ImagePackagePullContext( + projectId = projectId, + repoName = repoName, + remoteUrl = remoteUrl, + userName = it.username, + password = it.password + )) + } catch (e: Exception) { + logger.warn("illegal proxy url ${it.url} for repo $projectId|$repoName") + } } } else -> throw UnsupportedOperationException() } + return result } private fun buildString(stageTag: List): String { From 502841fa9676529f46bc4dbb9b256d286ccc7e61 Mon Sep 17 00:00:00 2001 From: zacYL Date: Mon, 3 Jul 2023 21:57:01 +0800 Subject: [PATCH 06/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/bkrepo/job/batch/RepoInitJob.kt | 29 +++++++++---------- .../bkrepo/job/batch/RepoRefreshJob.kt | 29 +++++++++---------- .../service/impl/OciOperationServiceImpl.kt | 3 +- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt index 310bb3b81a..d796260574 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt @@ -77,24 +77,21 @@ class RepoInitJob( } override fun run(row: ProxyRepoData, collectionName: String, context: JobContext) { - with(row) { - try { - val config = configuration.readJsonString() - if (checkConfigType(config)) { - logger.info("init request will be sent in repo $projectId|$name") - when (row.type) { - RepositoryType.HELM.name -> { - helmClient.initIndexAndPackage(projectId, name) - } - RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { - ociClient.getPackagesFromThirdPartyRepo(projectId, name) - } - else -> throw UnsupportedOperationException() - } + try { + val config = row.configuration.readJsonString() + if (!checkConfigType(config)) return + logger.info("init request will be sent in repo ${row.projectId}|${row.name}") + when (row.type) { + RepositoryType.HELM.name -> { + helmClient.initIndexAndPackage(row.projectId, row.name) } - } catch (e: Exception) { - throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) + RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { + ociClient.getPackagesFromThirdPartyRepo(row.projectId, row.name) + } + else -> throw UnsupportedOperationException() } + } catch (e: Exception) { + throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt index bbccf869a7..b786b23a8a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt @@ -73,24 +73,21 @@ class RepoRefreshJob( } override fun run(row: ProxyRepoData, collectionName: String, context: JobContext) { - with(row) { - try { - val config = configuration.readJsonString() - if (checkConfigType(config)) { - logger.info("Refresh request will be sent in repo $projectId|$name") - when (row.type) { - RepositoryType.HELM.name -> { - helmClient.refreshIndexYamlAndPackage(projectId, name) - } - RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { - ociClient.getPackagesFromThirdPartyRepo(projectId, name) - } - else -> throw UnsupportedOperationException() - } + try { + val config = row.configuration.readJsonString() + if (!checkConfigType(config)) return + logger.info("Refresh request will be sent in repo ${row.projectId}|${row.name}") + when (row.type) { + RepositoryType.HELM.name -> { + helmClient.refreshIndexYamlAndPackage(row.projectId, row.name) } - } catch (e: Exception) { - throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) + RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { + ociClient.getPackagesFromThirdPartyRepo(row.projectId, row.name) + } + else -> throw UnsupportedOperationException() } + } catch (e: Exception) { + throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index ae4c17cd8a..2d66114e63 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -1104,7 +1104,8 @@ class OciOperationServiceImpl( } override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String,) { - val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") + val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") buildImagePackagePullContext(projectId, repoName, repositoryDetail.configuration).forEach { pluginManager.applyExtension { queryAndCreateDockerPackageInfo(it) From f40145d260fb006daf2cf0563e3cb04a806c9cc6 Mon Sep 17 00:00:00 2001 From: zacYL Date: Mon, 3 Jul 2023 22:03:50 +0800 Subject: [PATCH 07/41] =?UTF-8?q?bug:=20=E5=BC=95=E7=94=A8=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../base/impl/remote/type/helm/HelmArtifactPushClient.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt index 3639472d13..2e57791dc0 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.BasicAuthUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.service.cluster.ClusterInfo import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager @@ -40,7 +41,6 @@ import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext import com.tencent.bkrepo.replication.replica.base.handler.DefaultHandler import com.tencent.bkrepo.replication.replica.base.impl.remote.base.PushClient -import com.tencent.bkrepo.replication.util.HttpUtils import com.tencent.bkrepo.repository.pojo.node.NodeDetail import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.MultipartBody @@ -258,7 +258,7 @@ class HelmArtifactPushClient( ): String { val baseUrl = URL(url) val v2Url = URL(baseUrl, baseUrl.path) - return HttpUtils.buildUrl(v2Url.toString(), path, params) + return UrlFormatter.buildUrl(v2Url.toString(), path, params) } private fun buildAuthRequestProperties(clusterInfo: ClusterInfo): RequestProperty { From 07083faeb3c1d02db392290057605d6be701a7fb Mon Sep 17 00:00:00 2001 From: zacYL Date: Tue, 4 Jul 2023 15:26:23 +0800 Subject: [PATCH 08/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/remote/RemoteRepository.kt | 1 + .../repository/OciRegistryRemoteRepository.kt | 53 ++++++++++--------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt index 5baf2589b5..7f9e3ffd5f 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt @@ -233,6 +233,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() { if (addInterceptor) { createAuthenticateInterceptor(configuration.credentials)?.let { builder.addInterceptor(it) } } + builder.retryOnConnectionFailure(true) return builder.build() } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index b89a7ba6c4..60aa09c30f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -45,6 +45,7 @@ import com.tencent.bkrepo.common.api.util.BasicAuthUtils import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -151,32 +152,31 @@ class OciRegistryRemoteRepository( logger.info("Remote request $downloadUrl will be sent") val tokenKey = buildTokenCacheKey(context.getStringAttribute(PROXY_URL)!!, property.imageName) val request = buildRequest(downloadUrl, remoteConfiguration, tokenCache.getIfPresent(tokenKey)) - val response = httpClient!!.newCall(request).execute() - var responseWithAuth: Response? = null try { - if (response.isSuccessful) return onResponse(context, response) - // 针对返回401进行token获取 - val token = getAuthenticationCode(context, response, remoteConfiguration, property.imageName) - if (token.isNullOrBlank()) return null - tokenCache.put(tokenKey, token) - val requestWithToken = buildRequest( - url = downloadUrl, - configuration = remoteConfiguration, - addBasicInterceptor = false, - token = token - ) - responseWithAuth = httpClient - .newCall(requestWithToken) - .execute() - return if (checkResponse(responseWithAuth)) { - onResponse(context, responseWithAuth) - } else null - } finally { - response.body?.close() - responseWithAuth?.body?.close() + httpClient!!.newCall(request).execute().use { + if (it.isSuccessful) return onResponse(context, it) + // 针对返回401进行token获取 + val token = getAuthenticationCode(context, it, remoteConfiguration, property.imageName) ?: return null + tokenCache.put(tokenKey, token) + val requestWithToken = buildRequest( + url = downloadUrl, + configuration = remoteConfiguration, + addBasicInterceptor = false, + token = token + ) + httpClient.newCall(requestWithToken).execute().use {responseWithAuth -> + return if (checkResponse(responseWithAuth)) { + onResponse(context, responseWithAuth) + } else null + } + } + } catch (e: Exception) { + logger.error("Error occurred while sending request $downloadUrl", e) + throw NodeNotFoundException(downloadUrl) } } + private fun onResponse(context: ArtifactContext, response: Response): Any? { if (context is ArtifactDownloadContext) { return onDownloadResponse(context, response) @@ -314,17 +314,19 @@ class OciRegistryRemoteRepository( imageName: String ): String? { if (response.code != HttpStatus.UNAUTHORIZED.value) { + logger.warn("response code is ${response.code} for url ${response.request.url}") return null } val wwwAuthenticate = response.header(WWW_AUTHENTICATE) if (wwwAuthenticate.isNullOrBlank() || !wwwAuthenticate.startsWith(BEARER_AUTH_PREFIX)) { + logger.warn("response wwwAuthenticate header $wwwAuthenticate is illegal") return null } val url = context.getStringAttribute(PROXY_URL)!! val scope = getScope(url, imageName) val authProperty = AuthenticationUtil.parseWWWAuthenticateHeader(wwwAuthenticate, scope) if (authProperty == null) { - logger.warn("Auth url can not be parsed from header!") + logger.warn("Auth url can not be parsed from header $wwwAuthenticate!") return null } val urlStr = AuthenticationUtil.buildAuthenticationUrl(authProperty, configuration.credentials.username) @@ -346,7 +348,7 @@ class OciRegistryRemoteRepository( " code is ${it.code} and response is $error" logger.warn(errMsg) throw ErrorCodeException( - OciMessageCode.OCI_REMOTE_CONFIGURATION_ERROR, + OciMessageCode.OCI_REMOTE_CREDENTIALS_INVALID, errMsg ) } @@ -360,7 +362,6 @@ class OciRegistryRemoteRepository( ) } } - } /** @@ -405,7 +406,7 @@ class OciRegistryRemoteRepository( inputStream = artifactStream, artifactName = context.artifactInfo.getResponseName(), node = node, - channel = ArtifactChannel.LOCAL + channel = ArtifactChannel.PROXY ) return buildResponse( cacheNode = node, From d54673b5ba0bcde6e45104059cbe6e9484045da6 Mon Sep 17 00:00:00 2001 From: zacYL Date: Tue, 4 Jul 2023 15:53:01 +0800 Subject: [PATCH 09/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/OciRegistryRemoteRepository.kt | 64 ++++++++++++------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 60aa09c30f..606e5796d0 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -155,20 +155,18 @@ class OciRegistryRemoteRepository( try { httpClient!!.newCall(request).execute().use { if (it.isSuccessful) return onResponse(context, it) - // 针对返回401进行token获取 - val token = getAuthenticationCode(context, it, remoteConfiguration, property.imageName) ?: return null - tokenCache.put(tokenKey, token) - val requestWithToken = buildRequest( - url = downloadUrl, - configuration = remoteConfiguration, - addBasicInterceptor = false, - token = token - ) - httpClient.newCall(requestWithToken).execute().use {responseWithAuth -> - return if (checkResponse(responseWithAuth)) { - onResponse(context, responseWithAuth) - } else null + if (it.code != HttpStatus.UNAUTHORIZED.value) { + logger.warn("response code is ${it.code} for url $downloadUrl") + return null } + return doRequestWithToken( + context = context, + wwwAuthenticate = it.header(WWW_AUTHENTICATE), + remoteConfiguration = remoteConfiguration, + imageName = property.imageName, + tokenKey = tokenKey, + downloadUrl = downloadUrl + ) } } catch (e: Exception) { logger.error("Error occurred while sending request $downloadUrl", e) @@ -176,6 +174,34 @@ class OciRegistryRemoteRepository( } } + /** + * 当返回401时，按照docker标准协议去拉取token，然后进行文件下载 + */ + private fun doRequestWithToken( + context: ArtifactContext, + wwwAuthenticate: String?, + remoteConfiguration: RemoteConfiguration, + imageName: String, + tokenKey: String, + downloadUrl: String, + ): Any? { + // 针对返回401进行token获取 + val proxyUrl = context.getStringAttribute(PROXY_URL)!! + val token = getAuthenticationCode(proxyUrl, wwwAuthenticate, remoteConfiguration, imageName) ?: return null + tokenCache.put(tokenKey, token) + val requestWithToken = buildRequest( + url = downloadUrl, + configuration = remoteConfiguration, + addBasicInterceptor = false, + token = token + ) + clientCache.getIfPresent(remoteConfiguration)!!.newCall(requestWithToken).execute().use {responseWithAuth -> + return if (checkResponse(responseWithAuth)) { + onResponse(context, responseWithAuth) + } else null + } + } + private fun onResponse(context: ArtifactContext, response: Response): Any? { if (context is ArtifactDownloadContext) { @@ -308,22 +334,16 @@ class OciRegistryRemoteRepository( } private fun getAuthenticationCode( - context: ArtifactContext, - response: Response, + proxyUrl: String, + wwwAuthenticate: String?, configuration: RemoteConfiguration, imageName: String ): String? { - if (response.code != HttpStatus.UNAUTHORIZED.value) { - logger.warn("response code is ${response.code} for url ${response.request.url}") - return null - } - val wwwAuthenticate = response.header(WWW_AUTHENTICATE) if (wwwAuthenticate.isNullOrBlank() || !wwwAuthenticate.startsWith(BEARER_AUTH_PREFIX)) { logger.warn("response wwwAuthenticate header $wwwAuthenticate is illegal") return null } - val url = context.getStringAttribute(PROXY_URL)!! - val scope = getScope(url, imageName) + val scope = getScope(proxyUrl, imageName) val authProperty = AuthenticationUtil.parseWWWAuthenticateHeader(wwwAuthenticate, scope) if (authProperty == null) { logger.warn("Auth url can not be parsed from header $wwwAuthenticate!") From f492765d7d967091b8b67e89ab503d6ccfe82ec5 Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 5 Jul 2023 11:39:50 +0800 Subject: [PATCH 10/41] =?UTF-8?q?bug:=20token=E7=BC=93=E5=AD=98key?= =?UTF-8?q?=E8=B0=83=E6=95=B4=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../artifact/repository/OciRegistryRemoteRepository.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 606e5796d0..5b78987e8e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -150,7 +150,7 @@ class OciRegistryRemoteRepository( val property = getRemoteRequestProperty(context) val downloadUrl = createRemoteDownloadUrl(context, property) logger.info("Remote request $downloadUrl will be sent") - val tokenKey = buildTokenCacheKey(context.getStringAttribute(PROXY_URL)!!, property.imageName) + val tokenKey = buildTokenCacheKey(context.getStringAttribute(PROXY_URL)!!,remoteConfiguration.credentials.username, property.imageName) val request = buildRequest(downloadUrl, remoteConfiguration, tokenCache.getIfPresent(tokenKey)) try { httpClient!!.newCall(request).execute().use { @@ -350,7 +350,7 @@ class OciRegistryRemoteRepository( return null } val urlStr = AuthenticationUtil.buildAuthenticationUrl(authProperty, configuration.credentials.username) - logger.info("The url for authenticating is $urlStr") + logger.info("The url for authentication is $urlStr") if (urlStr.isNullOrEmpty()) return null val request = buildRequest( url = urlStr, @@ -602,9 +602,9 @@ class OciRegistryRemoteRepository( return n } - private fun buildTokenCacheKey(remoteUrl: String, imageName: String): String { + private fun buildTokenCacheKey(remoteUrl: String, userName: String?, imageName: String): String { val scope = getScope(remoteUrl, imageName) - return "$remoteUrl${CharPool.COLON}$scope" + return "$remoteUrl${CharPool.COLON}$scope${CharPool.COLON}$userName" } companion object { From 32ebeba91dbeb4aedb6a3eba13764255176a9ee6 Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 5 Jul 2023 14:48:23 +0800 Subject: [PATCH 11/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/artifact/util/http/UrlFormatter.kt | 14 +++- .../artifact/util/http/UrlFormatterTest.kt | 75 +++++++++++++++++++ .../bkrepo/replication/util/HttpUtils.kt | 4 +- 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt index 803642cd94..6b540cda8c 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt @@ -102,15 +102,21 @@ object UrlFormatter { throw IllegalArgumentException("Url should not be blank") val newUrl = addProtocol(url.trim().trimEnd(SLASH)) val baseUrl = URL(newUrl, newUrl.path) - val baseParams = newUrl.query val builder = StringBuilder(baseUrl.toString().trimEnd(SLASH)) if (path.isNotBlank()) { builder.append(SLASH).append(path.trimStart(SLASH)) } - if (!baseParams.isNullOrEmpty()) { - builder.append(QUESTION).append(baseParams) + if (!newUrl.query.isNullOrEmpty()) { + builder.append(QUESTION).append(newUrl.query) } - if (params.isNotBlank()) { + return addParams(builder.toString(), params) + } + + fun addParams(url: String, params: String): String { + val baseUrl = URL(url) + val builder = StringBuilder(baseUrl.toString()) + + if (params.isNotEmpty()) { if (builder.contains(QUESTION)) { builder.append(CharPool.AND).append(params) } else { diff --git a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt index 582bc435b5..1ddb218236 100644 --- a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt +++ b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt @@ -137,6 +137,81 @@ internal class UrlFormatterTest { ) } + @Test + fun buildUrl9() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2/" + + val result = UrlFormatter.buildUrl(url, path) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/", result)} + ) + } + + @Test + fun buildUrl10() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2/" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/?a=a", result)} + ) + } + + @Test + fun buildUrl11() { + val url = "http://bkrepo.example.com/test/?b=b" + val path = "/v2/" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/?b=b&a=a", result)} + ) + } + + @Test + fun addParams() { + val url = "http://bkrepo.example.com/test/?b=b" + val params = "a=a" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b&a=a", result)} + ) + } + + @Test + fun addParams1() { + val url = "http://bkrepo.example.com/test/?b=b" + val params = "" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b", result)} + ) + } + + @Test + fun addParams2() { + val url = "http://bkrepo.example.com/test/" + val params = "b=b" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b", result)} + ) + } + + @Test + fun addParams3() { + val url = "http://bkrepo.example.com/test" + val params = "b=b" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test?b=b", result)} + ) + } + + + @Test fun addProtocol() { val url = "bkrepo.example.com/test/" diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt index b63779d48a..1010c53c09 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt @@ -28,7 +28,7 @@ package com.tencent.bkrepo.replication.util import com.tencent.bkrepo.common.api.constant.HttpHeaders -import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter.buildUrl +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter.addParams import com.tencent.bkrepo.replication.pojo.blob.RequestTag import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import okhttp3.Request @@ -46,7 +46,7 @@ object HttpUtils { ): Request { with(requestProperty) { val url = params?.let { - buildUrl(url = requestUrl!!, params = params!!) + addParams(url = requestUrl!!, params = params!!) } ?: requestUrl!! var builder = Request.Builder() .url(url) From 7ac81bb3e64deec3c9a937406a80c9948398c2ec Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 5 Jul 2023 14:53:16 +0800 Subject: [PATCH 12/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#462?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/artifact/repository/OciRegistryRemoteRepository.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 5b78987e8e..f4005cccc6 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -150,7 +150,9 @@ class OciRegistryRemoteRepository( val property = getRemoteRequestProperty(context) val downloadUrl = createRemoteDownloadUrl(context, property) logger.info("Remote request $downloadUrl will be sent") - val tokenKey = buildTokenCacheKey(context.getStringAttribute(PROXY_URL)!!,remoteConfiguration.credentials.username, property.imageName) + val tokenKey = buildTokenCacheKey( + context.getStringAttribute(PROXY_URL)!!,remoteConfiguration.credentials.username, property.imageName + ) val request = buildRequest(downloadUrl, remoteConfiguration, tokenCache.getIfPresent(tokenKey)) try { httpClient!!.newCall(request).execute().use { From a6419f1f3998e352fa4fda1e9e260d299e40f159 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Wed, 5 Jul 2023 17:53:26 +0800 Subject: [PATCH 13/41] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/OciOperationServiceImpl.kt | 220 +++++++++--------- 1 file changed, 111 insertions(+), 109 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 2d66114e63..ac8158f0c8 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -111,6 +111,11 @@ import com.tencent.devops.plugin.api.applyExtension import org.apache.commons.lang3.StringUtils import org.slf4j.Logger import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where import org.springframework.stereotype.Service import java.nio.charset.Charset import java.time.Instant @@ -558,27 +563,22 @@ class OciOperationServiceImpl( "Will start to update oci info for ${ociArtifactInfo.getArtifactFullPath()} " + "in repo ${ociArtifactInfo.getRepoIdentify()}" ) - val manifestBytes = storageService.load( - nodeDetail.sha256.orEmpty(), - Range.full(nodeDetail.size), - storageCredentials - )!!.readText() - val schemaVersion = OciUtils.schemeVersion(manifestBytes) + + val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) // 将该版本对应的blob sha256放到manifest节点的元数据中 - var digestList: List? = null - val (mediaType, manifest) = if (schemaVersion.schemaVersion == 1) { + val (mediaType, digestList) = if (manifest == null) { Pair(DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1, null) } else { - val manifest = OciUtils.stringToManifestV2(manifestBytes) // 更新manifest文件的metadata val mediaTypeV2 = if (manifest.mediaType.isNullOrEmpty()) { HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE } else { manifest.mediaType } - digestList = OciUtils.manifestIteratorDigest(manifest) - Pair(mediaTypeV2, manifest) + Pair(mediaTypeV2, OciUtils.manifestIteratorDigest(manifest)) } + + // 更新manifest节点元数据 updateNodeMetaData( projectId = ociArtifactInfo.projectId, repoName = ociArtifactInfo.repoName, @@ -588,25 +588,35 @@ class OciOperationServiceImpl( digestList = digestList, sourceType = sourceType ) - // 同步blob相关metadata - if (ociArtifactInfo.packageName.isNotEmpty()) { - if (schemaVersion.schemaVersion == 1) { - syncBlobInfoV1( - ociArtifactInfo = ociArtifactInfo, - manifestDigest = digest, - manifestPath = nodeDetail.fullPath, - sourceType = sourceType - ) - } else { - syncBlobInfo( - ociArtifactInfo = ociArtifactInfo, - manifest = manifest!!, - manifestDigest = digest, - storageCredentials = storageCredentials, - manifestPath = nodeDetail.fullPath, - sourceType = sourceType - ) - } + + + if (ociArtifactInfo.packageName.isEmpty()) return + // 处理manifest中的blob数据 + syncBlobInfo( + ociArtifactInfo = ociArtifactInfo, + manifest = manifest, + storageCredentials = storageCredentials, + nodeDetail = nodeDetail, + sourceType = sourceType + ) + } + + + private fun loadManifest( + sha256: String, + size: Long, + storageCredentials: StorageCredentials? + ): ManifestSchema2? { + val manifestBytes = storageService.load( + sha256, + Range.full(size), + storageCredentials + )!!.readText() + + return try { + OciUtils.stringToManifestV2(manifestBytes) + } catch (e: OciBadRequestException) { + null } } @@ -617,34 +627,17 @@ class OciOperationServiceImpl( with(ociArtifactInfo) { val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: return false val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return false - val manifestBytes = storageService.load( - nodeDetail.sha256!!, - Range.full(nodeDetail.size), - repositoryDetail.storageCredentials - )!!.readText() - val manifest = OciUtils.stringToManifestV2(manifestBytes) - val descriptorList = OciUtils.manifestIterator(manifest) - // 用于判断是否所有blob都以存在 - var existFlag: Boolean - var size: Long = 0 - - descriptorList.forEach { - size += it.size - existFlag = doSyncBlob(it, ociArtifactInfo, repositoryDetail.storageCredentials) - // 如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 - if (!existFlag) return false - } - // 根据flag生成package信息以及package version信息 - doPackageOperations( - manifestPath = manifestPath, + val manifest = loadManifest( + nodeDetail.sha256!!, nodeDetail.size, repositoryDetail.storageCredentials + ) ?: return false + return syncBlobInfo( ociArtifactInfo = ociArtifactInfo, - manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), - size = size, - chartYaml = null, + manifest = manifest, + storageCredentials = repositoryDetail.storageCredentials, + nodeDetail = nodeDetail, sourceType = ArtifactChannel.REPLICATION, userId = SYSTEM_USER ) - return true } } @@ -678,54 +671,77 @@ class OciOperationServiceImpl( ) } + /** - * 同步fsLayers层的数据 + * 同步blob层的数据和config里面的数据 */ - private fun syncBlobInfoV1( + private fun syncBlobInfo( ociArtifactInfo: OciManifestArtifactInfo, - manifestDigest: OciDigest, - manifestPath: String, - sourceType: ArtifactChannel? = null - ) { + nodeDetail: NodeDetail, + sourceType: ArtifactChannel? = null, + manifest: ManifestSchema2? = null, + storageCredentials: StorageCredentials?, + userId: String = SecurityUtils.getUserId() + ): Boolean { logger.info( - "Will start to sync fsLayers' blob info from manifest ${ociArtifactInfo.getArtifactFullPath()} " + - "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." + "Will start to sync blobs and config info from manifest ${ociArtifactInfo.getArtifactFullPath()} " + + "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." ) - // 根据flag生成package信息以及packageversion信息 - doPackageOperations( - manifestPath = manifestPath, - ociArtifactInfo = ociArtifactInfo, - manifestDigest = manifestDigest, - size = 0, - sourceType = sourceType + // existFlag 判断manifest里的所有blob是否都已经创建节点 + // size 整个镜像blob汇总的大小 + // yamlContent 当为helm v3版本的chart包时，读取yaml内容 + val (existFlag, size, yamlContent) = manifestHandler( + manifest, ociArtifactInfo, storageCredentials ) + // 如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 + if (existFlag) { + // 第三方同步的索引更新等所有文件全部上传完成后才去进行 + // 根据flag生成package信息以及package version信息 + doPackageOperations( + manifestPath = nodeDetail.fullPath, + ociArtifactInfo = ociArtifactInfo, + manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), + size = size, + yamlContent = yamlContent, + sourceType = sourceType, + userId = userId + ) + return true + } else { + val query = Query.query( + where(TOciReplicationRecord::projectId).isEqualTo(ociArtifactInfo.projectId) + .and(TOciReplicationRecord::repoName).isEqualTo(ociArtifactInfo.repoName) + .and(TOciReplicationRecord::packageName).isEqualTo(ociArtifactInfo.packageName) + .and(TOciReplicationRecord::packageVersion).isEqualTo(ociArtifactInfo.reference) + ) + val update = Update().setOnInsert(TOciReplicationRecord::manifestPath.name, nodeDetail.fullPath) + ociReplicationRecordDao.upsert(query, update) + return false + } } + /** - * 同步blob层的数据和config里面的数据 + * 针对v2版本manifest文件做特殊处理 */ - private fun syncBlobInfo( + private fun manifestHandler( + manifest: ManifestSchema2?, ociArtifactInfo: OciManifestArtifactInfo, - manifest: ManifestSchema2, - manifestDigest: OciDigest, - storageCredentials: StorageCredentials?, - manifestPath: String, - sourceType: ArtifactChannel? = null - ) { - logger.info( - "Will start to sync blobs and config info from manifest ${ociArtifactInfo.getArtifactFullPath()} " + - "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." - ) - val descriptorList = OciUtils.manifestIterator(manifest) + storageCredentials: StorageCredentials? + ): Triple?> { + //当manifest为空时，可能是v1版本镜像,直接返回 + if (manifest == null) return Triple(true, 0, null) // 用于判断是否所有blob都以存在 var existFlag = true - var chartYaml: Map? = null - // 统计所有manifest中的文件size作为整个package version的size + // 统计所有mainfest中的文件size作为整个package version的size var size: Long = 0 + val descriptorList = OciUtils.manifestIterator(manifest) + // 用于收集helm chart v3的yaml内容 + var yamlContent: Map? = null // 同步layer以及config层blob信息 descriptorList.forEach { size += it.size - chartYaml = when (it.mediaType) { + yamlContent = when (it.mediaType) { CHART_LAYER_MEDIA_TYPE -> { // 针对helm chart，需要将chart.yaml中相关信息存入对应节点中 loadArtifactInput( @@ -740,29 +756,15 @@ class OciOperationServiceImpl( else -> null } existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, storageCredentials) + if (!existFlag) { + // 第三方同步场景下，如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 + return Triple(false, 0, null) + } } - // 如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 - if (existFlag) { - // 根据flag生成package信息以及package version信息 - doPackageOperations( - manifestPath = manifestPath, - ociArtifactInfo = ociArtifactInfo, - manifestDigest = manifestDigest, - size = size, - chartYaml = chartYaml, - sourceType = sourceType - ) - } else { - ociReplicationRecordDao.save(TOciReplicationRecord( - projectId = ociArtifactInfo.projectId, - repoName = ociArtifactInfo.repoName, - packageName = ociArtifactInfo.packageName, - packageVersion = ociArtifactInfo.reference, - manifestPath = manifestPath - )) - } + return Triple(existFlag, size, yamlContent) } + /** * 更新blobs的信息 */ @@ -827,7 +829,7 @@ class OciOperationServiceImpl( ociArtifactInfo: OciManifestArtifactInfo, manifestDigest: OciDigest, size: Long, - chartYaml: Map? = null, + yamlContent: Map? = null, sourceType: ArtifactChannel? = null, userId: String = SecurityUtils.getUserId() ) { @@ -838,7 +840,7 @@ class OciOperationServiceImpl( val packageKey = PackageKeys.ofName(repoType.toLowerCase(), packageName) val metadata = mutableMapOf(MANIFEST_DIGEST to manifestDigest.toString()) .apply { - chartYaml?.let { this.putAll(chartYaml) } + yamlContent?.let { this.putAll(yamlContent) } sourceType?.let { this[SOURCE_TYPE] = sourceType } } val request = ObjectBuildUtils.buildPackageVersionCreateRequest( @@ -860,8 +862,8 @@ class OciOperationServiceImpl( ) // 针对helm chart包，将部分信息放入到package中 - chartYaml?.let { - val (appVersion, description) = getMetaDataFromChart(chartYaml) + yamlContent?.let { + val (appVersion, description) = getMetaDataFromChart(yamlContent) updatePackageInfo( ociArtifactInfo = ociArtifactInfo, appVersion = appVersion, From 38f41394067e70dc73e1dbec0882acb4e24abc27 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 10:39:28 +0800 Subject: [PATCH 14/41] =?UTF-8?q?feat:=20blob=E5=AD=98=E5=82=A8=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E8=B7=AF=E5=BE=84=E8=B0=83=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/util/OciLocationUtils.kt | 4 ++ .../service/impl/OciOperationServiceImpl.kt | 51 +++++++++++-------- .../bkrepo/oci/util/ObjectBuildUtils.kt | 5 +- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index 550222ab67..adb897cc89 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -86,6 +86,10 @@ object OciLocationUtils { } } + fun blobVersionPathLocation(reference: String, packageName: String, fileName: String): String { + return "/$packageName/blobs/$reference/"+ fileName + } + private fun returnPathLocation(digest: OciDigest, ociArtifactInfo: OciArtifactInfo, type: String): String { with(ociArtifactInfo) { return "/$packageName/$type/$digest" diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index ac8158f0c8..9f1d287376 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -691,7 +691,7 @@ class OciOperationServiceImpl( // size 整个镜像blob汇总的大小 // yamlContent 当为helm v3版本的chart包时，读取yaml内容 val (existFlag, size, yamlContent) = manifestHandler( - manifest, ociArtifactInfo, storageCredentials + manifest, ociArtifactInfo, storageCredentials, userId ) // 如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 if (existFlag) { @@ -727,7 +727,8 @@ class OciOperationServiceImpl( private fun manifestHandler( manifest: ManifestSchema2?, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials? + storageCredentials: StorageCredentials?, + userId: String = SecurityUtils.getUserId() ): Triple?> { //当manifest为空时，可能是v1版本镜像,直接返回 if (manifest == null) return Triple(true, 0, null) @@ -755,7 +756,7 @@ class OciOperationServiceImpl( } else -> null } - existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, storageCredentials) + existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, storageCredentials, userId) if (!existFlag) { // 第三方同步场景下，如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 return Triple(false, 0, null) @@ -771,7 +772,8 @@ class OciOperationServiceImpl( private fun doSyncBlob( descriptor: Descriptor, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials? + storageCredentials: StorageCredentials?, + userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { logger.info( @@ -787,7 +789,8 @@ class OciOperationServiceImpl( fullPath = fullPath, descriptor = descriptor, ociArtifactInfo = this, - storageCredentials = storageCredentials + storageCredentials = storageCredentials, + userId = userId ) } } @@ -799,24 +802,30 @@ class OciOperationServiceImpl( fullPath: String, descriptor: Descriptor, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials? + storageCredentials: StorageCredentials?, + userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { - val blobExist = nodeClient.checkExist(projectId, repoName, fullPath).data!! - // blob节点不存在，但是在同一仓库下存在digest文件 - if (!blobExist) { - val (existFullPath, md5) = getNodeByDigest(projectId, repoName, descriptor.digest) - if (existFullPath == null) return false - val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( - projectId = projectId, - repoName = repoName, - size = descriptor.size, - sha256 = descriptor.sha256, - fullPath = fullPath, - md5 = md5 ?: StringPool.UNKNOWN - ) - createNode(nodeCreateRequest, storageCredentials) - } + val nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) + if (nodeProperty.fullPath.isNullOrEmpty()) return false + val newPath = OciLocationUtils.blobVersionPathLocation(reference, packageName, descriptor.filename) + val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( + projectId = projectId, + repoName = repoName, + size = nodeProperty.size!!, + sha256 = descriptor.sha256, + fullPath = newPath, + md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, + userId = userId + ) + createNode(nodeCreateRequest, storageCredentials) + deleteNode( + projectId = projectId, + repoName = repoName, + packageName = packageName, + userId = userId, + path = fullPath + ) return true } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt index e5a9fd2bd9..61934b4a3f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt @@ -76,7 +76,8 @@ object ObjectBuildUtils { fullPath: String, sha256: String, md5: String, - metadata: List? = null + metadata: List? = null, + userId: String = SecurityUtils.getUserId() ): NodeCreateRequest { return NodeCreateRequest( projectId = projectId, @@ -86,7 +87,7 @@ object ObjectBuildUtils { size = size, sha256 = sha256, md5 = md5, - operator = SecurityUtils.getUserId(), + operator = userId, overwrite = true, nodeMetadata = metadata ) From d3c3d077fe1f8dcca4329f5810a71582eea0f806 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 11:38:51 +0800 Subject: [PATCH 15/41] =?UTF-8?q?feat:=20blob=E5=88=B7=E6=96=B0=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/pojo/artifact/OciArtifactInfo.kt | 1 + .../oci/controller/user/UserOciController.kt | 18 ++++++- .../bkrepo/oci/service/OciOperationService.kt | 8 ++- .../oci/service/impl/OciBlobServiceImpl.kt | 2 +- .../service/impl/OciOperationServiceImpl.kt | 51 ++++++++++++++----- 5 files changed, 65 insertions(+), 15 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt index 9229cc4541..c66265aedf 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt @@ -73,5 +73,6 @@ open class OciArtifactInfo( const val OCI_USER_REPO_SUFFIX = "/repo/{projectId}/{repoName}" const val OCI_USER_TAG_SUFFIX = "/tag/{projectId}/{repoName}/**" const val DOCKER_CATALOG_SUFFIX = "_catalog" + const val OCI_BLOB_NODE_FULLPATH_REFRESH = "/{projectId}/{repoName}/blob/node/refresh" } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt index 8394e2d629..aecbc3dc13 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt @@ -41,6 +41,7 @@ import com.tencent.bkrepo.oci.constant.PAGE_NUMBER import com.tencent.bkrepo.oci.constant.PAGE_SIZE import com.tencent.bkrepo.oci.constant.USER_API_PREFIX import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.OCI_BLOB_NODE_FULLPATH_REFRESH import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.OCI_PACKAGE_DELETE_URL import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.OCI_USER_MANIFEST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.OCI_USER_REPO_SUFFIX @@ -56,7 +57,6 @@ import com.tencent.bkrepo.oci.service.OciOperationService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam -import javax.servlet.http.HttpServletRequest import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable @@ -65,6 +65,7 @@ import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController import org.springframework.web.servlet.HandlerMapping +import javax.servlet.http.HttpServletRequest @Suppress("MVCPathVariableInspection") @Api("oci产品接口") @@ -201,4 +202,19 @@ class UserOciController( ) ) } + + @ApiOperation("刷新oci下的所有blob节点路径") + @GetMapping(OCI_BLOB_NODE_FULLPATH_REFRESH) + @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) + fun detailVersion( + @PathVariable + @ApiParam(value = OCI_PROJECT_ID, required = true) + projectId: String, + @PathVariable + @ApiParam(value = OCI_REPO_NAME, required = true) + repoName: String, + ): Response { + operationService.refreshOciBlobFullPath(projectId, repoName) + return ResponseBuilder.success() + } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index 3cb99c425d..335398343e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.oci.service import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel +import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo @@ -124,7 +125,7 @@ interface OciOperationService { /** * 当使用追加上传时，文件已存储，只需存储节点信息 */ - fun createNode(request: NodeCreateRequest, storageCredentials: StorageCredentials?): NodeDetail + fun createNode(request: NodeCreateRequest): NodeDetail /** * 保存文件内容(当使用追加上传时，文件已存储，只需存储节点信息) @@ -198,4 +199,9 @@ interface OciOperationService { * 拉取第三方镜像仓库package信息 */ fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String) + + /** + * oci blob 路径调整，由/packageName/blobs/XXX -> /packageName/blobs/version/XXX + */ + fun refreshOciBlobFullPath(projectId: String, repoName: String, userId: String = SecurityUtils.getUserId()) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index 10d5f5d0f6..d9ffba87a5 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -153,7 +153,7 @@ class OciBlobServiceImpl( md5 = nodeProperty.md5!! ) val repoDetail = repoClient.getRepoDetail(projectId, repoName).data - ociOperationService.createNode(nodeCreateRequest, repoDetail!!.storageCredentials) + ociOperationService.createNode(nodeCreateRequest) val blobLocation = OciLocationUtils.blobLocation(ociDigest, this) OciResponseUtils.buildBlobMountResponse( domain = domain, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 9f1d287376..a0a64f5846 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -311,7 +311,7 @@ class OciOperationServiceImpl( packageKey: String ) { with(artifactInfo) { - val nodeDetail = getBlobNodeDetail( + val nodeDetail = getImageNodeDetail( projectId = projectId, repoName = repoName, name = artifactInfo.packageName, @@ -335,7 +335,7 @@ class OciOperationServiceImpl( private fun findHelmChartYamlInfo(artifactInfo: OciArtifactInfo, version: String? = null): String? { with(artifactInfo) { if (version.isNullOrBlank()) return null - val nodeDetail = getBlobNodeDetail( + val nodeDetail = getImageNodeDetail( projectId = projectId, repoName = repoName, name = artifactInfo.packageName, @@ -365,7 +365,7 @@ class OciOperationServiceImpl( path: String? = null ) { val fullPath = path ?: digestStr?.let { - val nodeDetail = getBlobNodeDetail( + val nodeDetail = getImageNodeDetail( projectId = projectId, repoName = repoName, name = packageName, @@ -413,7 +413,7 @@ class OciOperationServiceImpl( packageKey = packageKey, version = version ) - val nodeDetail = getBlobNodeDetail( + val nodeDetail = getImageNodeDetail( projectId = projectId, repoName = repoName, name = name, @@ -431,7 +431,7 @@ class OciOperationServiceImpl( * 获取node节点 * 查不到抛出OciFileNotFoundException异常 */ - private fun getBlobNodeDetail( + private fun getImageNodeDetail( projectId: String, repoName: String, name: String, @@ -520,7 +520,7 @@ class OciOperationServiceImpl( md5 = fileInfo.md5, sha256 = fileInfo.sha256 ) - createNode(newNodeRequest, storageCredentials) + createNode(newNodeRequest) null } else { storageManager.storeArtifactFile(request, artifactFile, storageCredentials) @@ -540,7 +540,7 @@ class OciOperationServiceImpl( /** * 当使用追加上传时，文件已存储，只需存储节点信息 */ - override fun createNode(request: NodeCreateRequest, storageCredentials: StorageCredentials?): NodeDetail { + override fun createNode(request: NodeCreateRequest): NodeDetail { try { return nodeClient.createNode(request).data!! } catch (exception: Exception) { @@ -756,7 +756,7 @@ class OciOperationServiceImpl( } else -> null } - existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, storageCredentials, userId) + existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, userId) if (!existFlag) { // 第三方同步场景下，如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 return Triple(false, 0, null) @@ -772,7 +772,6 @@ class OciOperationServiceImpl( private fun doSyncBlob( descriptor: Descriptor, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials?, userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { @@ -789,7 +788,6 @@ class OciOperationServiceImpl( fullPath = fullPath, descriptor = descriptor, ociArtifactInfo = this, - storageCredentials = storageCredentials, userId = userId ) } @@ -802,7 +800,6 @@ class OciOperationServiceImpl( fullPath: String, descriptor: Descriptor, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials?, userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { @@ -818,7 +815,7 @@ class OciOperationServiceImpl( md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, userId = userId ) - createNode(nodeCreateRequest, storageCredentials) + createNode(nodeCreateRequest) deleteNode( projectId = projectId, repoName = repoName, @@ -1124,6 +1121,36 @@ class OciOperationServiceImpl( } } + override fun refreshOciBlobFullPath(projectId: String, repoName: String, userId: String) { + val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") + val packageList = packageClient.listAllPackageNames(projectId, repoName).data ?: return + packageList.forEach { pName -> + packageClient.listAllVersion(projectId, repoName, pName).data.orEmpty().forEach { pVersion -> + val packageName = PackageKeys.resolveName(repositoryDetail.type.name.toLowerCase(), pName) + val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion.name) + logger.info("Manifest $manifestPath will be refreshed") + val manifestNode = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return + val manifest = loadManifest(manifestNode.sha256!!, manifestNode.size, repositoryDetail.storageCredentials) + ?: run { + logger.warn("The content of manifest.json $manifestPath is null, check the mediaType.") + return + } + val ociArtifactInfo = OciManifestArtifactInfo( + projectId = projectId, + repoName = repoName, + packageName = packageName, + version = pVersion.name, + reference = pVersion.name, + isValidDigest = false + ) + OciUtils.manifestIterator(manifest).forEach { + doSyncBlob(it, ociArtifactInfo, userId) + } + } + } + } + private fun buildImagePackagePullContext( projectId: String, repoName: String, From d41898dce92f57189d9933476d31adaafff7e689 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 16:19:09 +0800 Subject: [PATCH 16/41] =?UTF-8?q?feat:=20=E5=8C=85=E4=B8=8E=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E5=88=A0=E9=99=A4=E9=80=BB=E8=BE=91=E8=B0=83=E6=95=B4?= =?UTF-8?q?=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/util/OciLocationUtils.kt | 14 +- .../bkrepo/oci/service/OciOperationService.kt | 12 - .../service/impl/OciOperationServiceImpl.kt | 276 +++++------------- .../bkrepo/oci/util/ObjectBuildUtils.kt | 4 - 4 files changed, 76 insertions(+), 230 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index adb897cc89..2b97fc3b1a 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -44,18 +44,10 @@ object OciLocationUtils { return buildDigestManifestPath(packageName, OciDigest(reference)) } - fun buildDigestManifestPathWithSha256(packageName: String, sha256: String): String { - return buildDigestManifestPath(packageName, OciDigest.fromSha256(sha256)) - } - private fun buildDigestManifestPath(packageName: String, ref: OciDigest): String { return buildPath(packageName, ref, "manifest") } - fun buildDigestBlobsPath(packageName: String, digestStr: String): String { - return buildPath(packageName, OciDigest(digestStr), "blobs") - } - fun buildDigestBlobsPath(packageName: String, ref: OciDigest): String { return buildPath(packageName, ref, "blobs") } @@ -87,7 +79,11 @@ object OciLocationUtils { } fun blobVersionPathLocation(reference: String, packageName: String, fileName: String): String { - return "/$packageName/blobs/$reference/"+ fileName + return blobVersionFolderLocation(reference, packageName)+ fileName + } + + fun blobVersionFolderLocation(reference: String, packageName: String): String { + return "/$packageName/blobs/$reference/" } private fun returnPathLocation(digest: OciDigest, ociArtifactInfo: OciArtifactInfo, type: String): String { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index 335398343e..22e79963a1 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -56,18 +56,6 @@ interface OciOperationService { userId: String ) - /** - * 当mediatype为CHART_LAYER_MEDIA_TYPE，需要解析chart.yaml文件 - */ - fun loadArtifactInput( - chartDigest: String?, - projectId: String, - repoName: String, - packageName: String, - version: String, - storageCredentials: StorageCredentials? - ): Map? - /** * 需要将blob中相关metadata写进package version中 */ diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index a0a64f5846..ac10838bac 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -52,8 +52,6 @@ import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties -import com.tencent.bkrepo.oci.constant.APP_VERSION -import com.tencent.bkrepo.oci.constant.CHART_LAYER_MEDIA_TYPE import com.tencent.bkrepo.oci.constant.DESCRIPTION import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 import com.tencent.bkrepo.oci.constant.DOWNLOADS @@ -177,37 +175,6 @@ class OciOperationServiceImpl( metadataClient.saveMetadata(metadataSaveRequest) } - /** - * 当mediaType为CHART_LAYER_MEDIA_TYPE，需要解析chart.yaml文件 - */ - override fun loadArtifactInput( - chartDigest: String?, - projectId: String, - repoName: String, - packageName: String, - version: String, - storageCredentials: StorageCredentials? - ): Map? { - if (chartDigest.isNullOrBlank()) return null - val blobDigest = OciDigest(chartDigest) - val fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, blobDigest) - nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { node -> - logger.info( - "Will read chart.yaml data from $fullPath with package $packageName " + - "and version $version under repo $projectId/$repoName" - ) - storageManager.loadArtifactInputStream(node, storageCredentials)?.let { - return try { - OciUtils.convertToMap(OciUtils.parseChartInputStream(it)) - } catch (e: Exception) { - logger.warn("Convert chart.yaml error: ${e.message}") - null - } - } - } - return null - } - /** * 需要将blob中相关metadata写进package version中 */ @@ -252,57 +219,20 @@ class OciOperationServiceImpl( ) } ?: throw VersionNotFoundException(version) } else { - packageClient.listAllVersion( + // 删除package目录 + deleteNode(projectId, repoName, "${StringPool.SLASH}$packageName", userId) + //删除package下所有版本 + packageClient.deletePackage( projectId, repoName, packageKey - ).data.orEmpty().forEach { - removeVersion( - artifactInfo = this, - version = it.name, - userId = userId, - packageKey = packageKey - ) - } - } - updatePackageExtension(artifactInfo, packageKey) - } - } - - /** - * 节点删除后，将package extension信息更新 - */ - private fun updatePackageExtension(artifactInfo: OciArtifactInfo, packageKey: String) { - with(artifactInfo) { - val version = packageClient.findPackageByKey(projectId, repoName, packageKey).data?.latest - try { - val chartDigest = findHelmChartYamlInfo(this, version) - val chartYaml = loadArtifactInput( - chartDigest = chartDigest, - projectId = projectId, - repoName = repoName, - packageName = packageName, - version = version!!, - storageCredentials = getRepositoryInfo(this).storageCredentials ) - // 针对helm chart包，将部分信息放入到package中 - chartYaml?.let { - val (appVersion, description) = getMetaDataFromChart(chartYaml) - updatePackageInfo( - ociArtifactInfo = artifactInfo, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - } - } catch (e: Exception) { - logger.warn("can not convert meta data") } } } /** - * 删除[version] 对应的node节点也会一起删除 + * 删除[version] 默认会删除对应的节点 */ private fun removeVersion( artifactInfo: OciArtifactInfo, @@ -311,78 +241,51 @@ class OciOperationServiceImpl( packageKey: String ) { with(artifactInfo) { - val nodeDetail = getImageNodeDetail( - projectId = projectId, - repoName = repoName, - name = artifactInfo.packageName, - version = version - ) ?: return - // 删除manifest - deleteNode( - projectId = projectId, - repoName = repoName, - packageName = packageName, - path = nodeDetail.fullPath, + // 删除对应版本下所有关联的节点，包含manifest以及blobs + deleteVersionRelatedNodes( + artifactInfo = artifactInfo, + version = version, userId = userId ) packageClient.deleteVersion(projectId, repoName, packageKey, version) } } + /** - * 针对helm特殊处理，查找chart对应的digest，用于读取对应的chart.yaml信息 + * 删除对应版本下所有关联的节点，包含manifest以及blobs */ - private fun findHelmChartYamlInfo(artifactInfo: OciArtifactInfo, version: String? = null): String? { + private fun deleteVersionRelatedNodes( + artifactInfo: OciArtifactInfo, + version: String, + userId: String, + ) { with(artifactInfo) { - if (version.isNullOrBlank()) return null - val nodeDetail = getImageNodeDetail( - projectId = projectId, - repoName = repoName, - name = artifactInfo.packageName, - version = version - ) ?: return null - val inputStream = storageService.load( - digest = nodeDetail.sha256!!, - range = Range.full(nodeDetail.size), - storageCredentials = getRepositoryInfo(artifactInfo).storageCredentials - ) ?: return null - try { - val manifest = OciUtils.streamToManifestV2(inputStream) - return (OciUtils.manifestIterator(manifest, CHART_LAYER_MEDIA_TYPE) ?: return null).digest - } catch (e: OciBadRequestException) { - logger.warn("Manifest convert error: ${e.message}") - } - return null + val manifest = OciLocationUtils.buildManifestPath(packageName, version) + val blobsFolder = OciLocationUtils.blobVersionFolderLocation(version, packageName) + logger.info("Will delete blobsFolder [$blobsFolder] and manifest $manifest " + + "in package $packageName|$version in repo [$projectId/$repoName]") + // 删除manifest + deleteNode(projectId, repoName, manifest, userId) + // 删除blobs + deleteNode(projectId, repoName, blobsFolder, userId) } } + private fun deleteNode( projectId: String, repoName: String, - packageName: String, - userId: String, - digestStr: String? = null, - path: String? = null + fullPath: String, + userId: String ) { - val fullPath = path ?: digestStr?.let { - val nodeDetail = getImageNodeDetail( - projectId = projectId, - repoName = repoName, - name = packageName, - digestStr = digestStr - ) - nodeDetail?.fullPath - } - fullPath?.let { - logger.info("Will delete node [$fullPath] with package $packageName in repo [$projectId/$repoName]") - val request = NodeDeleteRequest( - projectId = projectId, - repoName = repoName, - fullPath = fullPath, - operator = userId - ) - nodeClient.deleteNode(request) - } + val request = NodeDeleteRequest( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + operator = userId + ) + nodeClient.deleteNode(request) } /** @@ -595,7 +498,6 @@ class OciOperationServiceImpl( syncBlobInfo( ociArtifactInfo = ociArtifactInfo, manifest = manifest, - storageCredentials = storageCredentials, nodeDetail = nodeDetail, sourceType = sourceType ) @@ -633,7 +535,6 @@ class OciOperationServiceImpl( return syncBlobInfo( ociArtifactInfo = ociArtifactInfo, manifest = manifest, - storageCredentials = repositoryDetail.storageCredentials, nodeDetail = nodeDetail, sourceType = ArtifactChannel.REPLICATION, userId = SYSTEM_USER @@ -650,7 +551,6 @@ class OciOperationServiceImpl( version: String? = null, fullPath: String, mediaType: String, - chartYaml: Map? = null, digestList: List? = null, sourceType: ArtifactChannel? = null ) { @@ -658,7 +558,6 @@ class OciOperationServiceImpl( val metadata = ObjectBuildUtils.buildMetadata( mediaType = mediaType, version = version, - yamlData = chartYaml, digestList = digestList, sourceType = sourceType ) @@ -680,7 +579,6 @@ class OciOperationServiceImpl( nodeDetail: NodeDetail, sourceType: ArtifactChannel? = null, manifest: ManifestSchema2? = null, - storageCredentials: StorageCredentials?, userId: String = SecurityUtils.getUserId() ): Boolean { logger.info( @@ -689,9 +587,8 @@ class OciOperationServiceImpl( ) // existFlag 判断manifest里的所有blob是否都已经创建节点 // size 整个镜像blob汇总的大小 - // yamlContent 当为helm v3版本的chart包时，读取yaml内容 - val (existFlag, size, yamlContent) = manifestHandler( - manifest, ociArtifactInfo, storageCredentials, userId + val (existFlag, size) = manifestHandler( + manifest, ociArtifactInfo, userId ) // 如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 if (existFlag) { @@ -702,7 +599,6 @@ class OciOperationServiceImpl( ociArtifactInfo = ociArtifactInfo, manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), size = size, - yamlContent = yamlContent, sourceType = sourceType, userId = userId ) @@ -727,42 +623,32 @@ class OciOperationServiceImpl( private fun manifestHandler( manifest: ManifestSchema2?, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials?, userId: String = SecurityUtils.getUserId() - ): Triple?> { + ): Pair { //当manifest为空时，可能是v1版本镜像,直接返回 - if (manifest == null) return Triple(true, 0, null) + if (manifest == null) return Pair(true, 0) // 用于判断是否所有blob都以存在 var existFlag = true // 统计所有mainfest中的文件size作为整个package version的size var size: Long = 0 val descriptorList = OciUtils.manifestIterator(manifest) - // 用于收集helm chart v3的yaml内容 - var yamlContent: Map? = null + + // 当同一版本覆盖上传时，先删除当前版本对应的blobs目录，然后再创建 + val blobsFolder = OciLocationUtils.blobVersionFolderLocation( + ociArtifactInfo.reference, ociArtifactInfo.packageName + ) + deleteNode(ociArtifactInfo.projectId, ociArtifactInfo.repoName, blobsFolder, userId) + // 同步layer以及config层blob信息 descriptorList.forEach { size += it.size - yamlContent = when (it.mediaType) { - CHART_LAYER_MEDIA_TYPE -> { - // 针对helm chart，需要将chart.yaml中相关信息存入对应节点中 - loadArtifactInput( - chartDigest = it.digest, - projectId = ociArtifactInfo.projectId, - repoName = ociArtifactInfo.repoName, - packageName = ociArtifactInfo.packageName, - version = ociArtifactInfo.reference, - storageCredentials = storageCredentials - ) - } - else -> null - } existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, userId) if (!existFlag) { // 第三方同步场景下，如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 - return Triple(false, 0, null) + return Pair(false, 0) } } - return Triple(existFlag, size, yamlContent) + return Pair(existFlag, size) } @@ -803,26 +689,31 @@ class OciOperationServiceImpl( userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { - val nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) + // 并发情况下，版本目录下可能存在着非该版本的blob + // 覆盖上传时会先删除原有目录，并发情况下可能导致blobs节点不存在 + val nodeProperty = nodeClient.getDeletedNodeDetail( + projectId, repoName, fullPath + ).data?.firstOrNull{ it.sha256 == descriptor.sha256 }?.let { + NodeProperty(it.fullPath, it.md5, it.size) + } ?: getNodeByDigest(projectId, repoName, descriptor.digest) + if (nodeProperty.fullPath.isNullOrEmpty()) return false val newPath = OciLocationUtils.blobVersionPathLocation(reference, packageName, descriptor.filename) - val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( - projectId = projectId, - repoName = repoName, - size = nodeProperty.size!!, - sha256 = descriptor.sha256, - fullPath = newPath, - md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, - userId = userId - ) - createNode(nodeCreateRequest) - deleteNode( - projectId = projectId, - repoName = repoName, - packageName = packageName, - userId = userId, - path = fullPath - ) + if (newPath != nodeProperty.fullPath) { + // 创建新路径节点 /packageName/blobs/version/xxx + val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( + projectId = projectId, + repoName = repoName, + size = nodeProperty.size!!, + sha256 = descriptor.sha256, + fullPath = newPath, + md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, + userId = userId + ) + createNode(nodeCreateRequest) + } + // 删除临时存储路径节点 /packageName/blobs/xxx + deleteNode(projectId, repoName, fullPath, userId) return true } } @@ -835,7 +726,6 @@ class OciOperationServiceImpl( ociArtifactInfo: OciManifestArtifactInfo, manifestDigest: OciDigest, size: Long, - yamlContent: Map? = null, sourceType: ArtifactChannel? = null, userId: String = SecurityUtils.getUserId() ) { @@ -846,7 +736,6 @@ class OciOperationServiceImpl( val packageKey = PackageKeys.ofName(repoType.toLowerCase(), packageName) val metadata = mutableMapOf(MANIFEST_DIGEST to manifestDigest.toString()) .apply { - yamlContent?.let { this.putAll(yamlContent) } sourceType?.let { this[SOURCE_TYPE] = sourceType } } val request = ObjectBuildUtils.buildPackageVersionCreateRequest( @@ -866,17 +755,6 @@ class OciOperationServiceImpl( version = ociArtifactInfo.reference, metadata = metadata ) - - // 针对helm chart包，将部分信息放入到package中 - yamlContent?.let { - val (appVersion, description) = getMetaDataFromChart(yamlContent) - updatePackageInfo( - ociArtifactInfo = ociArtifactInfo, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - } } } @@ -901,18 +779,6 @@ class OciOperationServiceImpl( packageMetadataClient.saveMetadata(metadataSaveRequest) } - /** - * 针对helm chart包，将部分信息放入到package中 - */ - private fun getMetaDataFromChart(chartYaml: Map? = null): Pair { - var appVersion: String? = null - var description: String? = null - chartYaml?.let { - appVersion = it[APP_VERSION] as String? - description = it[DESCRIPTION] as String? - } - return Pair(appVersion, description) - } /** * 获取对应存储节点路径 @@ -1111,7 +977,7 @@ class OciOperationServiceImpl( return OciTagResult(result.totalRecords, data) } - override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String,) { + override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String) { val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") buildImagePackagePullContext(projectId, repoName, repositoryDetail.configuration).forEach { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt index 61934b4a3f..b1132e1d7a 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt @@ -96,7 +96,6 @@ object ObjectBuildUtils { fun buildMetadata( mediaType: String, version: String?, - yamlData: Map? = null, digestList: List? = null, sourceType: ArtifactChannel? = null ): MutableMap { @@ -106,9 +105,6 @@ object ObjectBuildUtils { version?.let { this.put(IMAGE_VERSION, version) } digestList?.let { this.put(DIGEST_LIST, digestList) } sourceType?.let { this.put(SOURCE_TYPE, sourceType) } - yamlData?.let { - this.putAll(yamlData) - } } } From d9372c0fa6effe0de4851b14ceab7c28bccbaadc Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 16:26:39 +0800 Subject: [PATCH 17/41] =?UTF-8?q?feat:=20=E6=96=B9=E6=B3=95=E5=90=8D?= =?UTF-8?q?=E8=B0=83=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/bkrepo/oci/controller/user/UserOciController.kt | 4 ++-- .../com/tencent/bkrepo/oci/service/OciOperationService.kt | 2 +- .../bkrepo/oci/service/impl/OciOperationServiceImpl.kt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt index aecbc3dc13..18fcf3e0b5 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt @@ -206,7 +206,7 @@ class UserOciController( @ApiOperation("刷新oci下的所有blob节点路径") @GetMapping(OCI_BLOB_NODE_FULLPATH_REFRESH) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) - fun detailVersion( + fun refreshFullPathOfBlob( @PathVariable @ApiParam(value = OCI_PROJECT_ID, required = true) projectId: String, @@ -214,7 +214,7 @@ class UserOciController( @ApiParam(value = OCI_REPO_NAME, required = true) repoName: String, ): Response { - operationService.refreshOciBlobFullPath(projectId, repoName) + operationService.refreshFullPathOfBlob(projectId, repoName) return ResponseBuilder.success() } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index 22e79963a1..28d7a7b6a1 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -191,5 +191,5 @@ interface OciOperationService { /** * oci blob 路径调整，由/packageName/blobs/XXX -> /packageName/blobs/version/XXX */ - fun refreshOciBlobFullPath(projectId: String, repoName: String, userId: String = SecurityUtils.getUserId()) + fun refreshFullPathOfBlob(projectId: String, repoName: String, userId: String = SecurityUtils.getUserId()) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index ac10838bac..120d0e8703 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -987,7 +987,7 @@ class OciOperationServiceImpl( } } - override fun refreshOciBlobFullPath(projectId: String, repoName: String, userId: String) { + override fun refreshFullPathOfBlob(projectId: String, repoName: String, userId: String) { val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") val packageList = packageClient.listAllPackageNames(projectId, repoName).data ?: return From e2bc7a6395e4bb3eff7c43b6fec83e71255639c2 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 17:40:03 +0800 Subject: [PATCH 18/41] =?UTF-8?q?feat:=20=E7=89=88=E6=9C=AC=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E8=B7=AF=E5=BE=84=E8=B0=83=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/oci/util/OciLocationUtils.kt | 9 +++++---- .../bkrepo/oci/service/impl/OciOperationServiceImpl.kt | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index 2b97fc3b1a..396232f9dd 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -30,14 +30,15 @@ package com.tencent.bkrepo.oci.util import com.tencent.bkrepo.oci.constant.OCI_MANIFEST import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest -import org.slf4j.LoggerFactory object OciLocationUtils { - private val logger = LoggerFactory.getLogger(OciLocationUtils::class.java) - fun buildManifestPath(packageName: String, tag: String): String { - return "/$packageName/manifest/$tag/$OCI_MANIFEST" + return buildManifestFolderPath(packageName, tag) + OCI_MANIFEST + } + + fun buildManifestFolderPath(packageName: String, tag: String): String { + return "/$packageName/manifest/$tag/" } fun buildDigestManifestPathWithReference(packageName: String, reference: String): String { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 120d0e8703..21279a75d5 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -261,12 +261,12 @@ class OciOperationServiceImpl( userId: String, ) { with(artifactInfo) { - val manifest = OciLocationUtils.buildManifestPath(packageName, version) + val manifestFolder = OciLocationUtils.buildManifestFolderPath(packageName, version) val blobsFolder = OciLocationUtils.blobVersionFolderLocation(version, packageName) - logger.info("Will delete blobsFolder [$blobsFolder] and manifest $manifest " + + logger.info("Will delete blobsFolder [$blobsFolder] and manifestFolder $manifestFolder " + "in package $packageName|$version in repo [$projectId/$repoName]") - // 删除manifest - deleteNode(projectId, repoName, manifest, userId) + // 删除manifestFolder + deleteNode(projectId, repoName, manifestFolder, userId) // 删除blobs deleteNode(projectId, repoName, blobsFolder, userId) } From 64c7b7dd726cb49d1f75a16dddc983e029850cd8 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 17:42:58 +0800 Subject: [PATCH 19/41] =?UTF-8?q?feat:=20=E6=8E=A5=E5=8F=A3=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/bkrepo/oci/controller/user/UserOciController.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt index 18fcf3e0b5..22b7d946c8 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt @@ -60,6 +60,7 @@ import io.swagger.annotations.ApiParam import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestAttribute import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam @@ -204,7 +205,7 @@ class UserOciController( } @ApiOperation("刷新oci下的所有blob节点路径") - @GetMapping(OCI_BLOB_NODE_FULLPATH_REFRESH) + @PostMapping(OCI_BLOB_NODE_FULLPATH_REFRESH) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun refreshFullPathOfBlob( @PathVariable From 43dae3046d67b4ea7ddf2c61212d79134b0a7ebc Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 17:56:09 +0800 Subject: [PATCH 20/41] =?UTF-8?q?feat:=20=E5=88=A0=E9=99=A4=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E8=B0=83=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/OciOperationServiceImpl.kt | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 21279a75d5..9257584eec 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -691,12 +691,14 @@ class OciOperationServiceImpl( with(ociArtifactInfo) { // 并发情况下，版本目录下可能存在着非该版本的blob // 覆盖上传时会先删除原有目录，并发情况下可能导致blobs节点不存在 - val nodeProperty = nodeClient.getDeletedNodeDetail( - projectId, repoName, fullPath - ).data?.firstOrNull{ it.sha256 == descriptor.sha256 }?.let { - NodeProperty(it.fullPath, it.md5, it.size) - } ?: getNodeByDigest(projectId, repoName, descriptor.digest) - + var nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) + if (nodeProperty.fullPath == null) { + nodeProperty = nodeClient.getDeletedNodeDetail( + projectId, repoName, fullPath + ).data?.firstOrNull{ it.sha256 == descriptor.sha256 }?.let { + NodeProperty(it.fullPath, it.md5, it.size) + } ?: return false + } if (nodeProperty.fullPath.isNullOrEmpty()) return false val newPath = OciLocationUtils.blobVersionPathLocation(reference, packageName, descriptor.filename) if (newPath != nodeProperty.fullPath) { @@ -997,10 +999,11 @@ class OciOperationServiceImpl( val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion.name) logger.info("Manifest $manifestPath will be refreshed") val manifestNode = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return - val manifest = loadManifest(manifestNode.sha256!!, manifestNode.size, repositoryDetail.storageCredentials) - ?: run { - logger.warn("The content of manifest.json $manifestPath is null, check the mediaType.") - return + val manifest = loadManifest( + manifestNode.sha256!!, manifestNode.size, repositoryDetail.storageCredentials + ) ?: run { + logger.warn("The content of manifest.json $manifestPath is null, check the mediaType.") + return } val ociArtifactInfo = OciManifestArtifactInfo( projectId = projectId, From fb38db29531e2fa979020615e8f22aff3a5483d7 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 20:14:52 +0800 Subject: [PATCH 21/41] =?UTF-8?q?feat:=20blob=E8=8A=82=E7=82=B9=E5=88=B7?= =?UTF-8?q?=E6=96=B0=E4=BB=A3=E7=A0=81=E8=B0=83=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/OciOperationServiceImpl.kt | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 9257584eec..edf4809aef 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -100,6 +100,7 @@ import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest +import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder @@ -995,31 +996,43 @@ class OciOperationServiceImpl( val packageList = packageClient.listAllPackageNames(projectId, repoName).data ?: return packageList.forEach { pName -> packageClient.listAllVersion(projectId, repoName, pName).data.orEmpty().forEach { pVersion -> - val packageName = PackageKeys.resolveName(repositoryDetail.type.name.toLowerCase(), pName) - val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion.name) - logger.info("Manifest $manifestPath will be refreshed") - val manifestNode = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return - val manifest = loadManifest( - manifestNode.sha256!!, manifestNode.size, repositoryDetail.storageCredentials - ) ?: run { - logger.warn("The content of manifest.json $manifestPath is null, check the mediaType.") - return - } - val ociArtifactInfo = OciManifestArtifactInfo( - projectId = projectId, - repoName = repoName, - packageName = packageName, - version = pVersion.name, - reference = pVersion.name, - isValidDigest = false - ) - OciUtils.manifestIterator(manifest).forEach { - doSyncBlob(it, ociArtifactInfo, userId) - } + refreshNode(pName, pVersion, repositoryDetail, userId) } } } + private fun refreshNode( + pName: String, + pVersion: PackageVersion, + repositoryDetail: RepositoryDetail, + userId: String + ) { + val packageName = PackageKeys.resolveName(repositoryDetail.type.name.toLowerCase(), pName) + val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion.name) + logger.info("Manifest $manifestPath will be refreshed") + val manifestNode = nodeClient.getNodeDetail( + repositoryDetail.projectId, repositoryDetail.name, manifestPath + ).data ?: return + val manifest = loadManifest( + manifestNode.sha256!!, manifestNode.size, repositoryDetail.storageCredentials + ) ?: run { + logger.warn("The content of manifest.json ${manifestNode.fullPath} is null, check the mediaType.") + return + } + val ociArtifactInfo = OciManifestArtifactInfo( + projectId = repositoryDetail.projectId, + repoName = repositoryDetail.name, + packageName = packageName, + version = pVersion.name, + reference = pVersion.name, + isValidDigest = false + ) + OciUtils.manifestIterator(manifest).forEach { + doSyncBlob(it, ociArtifactInfo, userId) + } + } + + private fun buildImagePackagePullContext( projectId: String, repoName: String, From 98d9f232c2318e392347069a8dbe17791c9178fe Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 20:20:54 +0800 Subject: [PATCH 22/41] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=89=93?= =?UTF-8?q?=E5=8D=B0=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index edf4809aef..752837f514 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -1030,6 +1030,7 @@ class OciOperationServiceImpl( OciUtils.manifestIterator(manifest).forEach { doSyncBlob(it, ociArtifactInfo, userId) } + logger.info("Manifest $manifestPath has been successfully refreshed") } From df3f3add80c12777e0a53b57909e32977ae315d9 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 22:09:28 +0800 Subject: [PATCH 23/41] =?UTF-8?q?feat:=20=E8=8E=B7=E5=8F=96=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E8=B7=AF=E5=BE=84=E8=B0=83=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/util/OciLocationUtils.kt | 10 ++-- .../oci/controller/user/OciBlobController.kt | 2 +- .../bkrepo/oci/service/OciOperationService.kt | 5 +- .../oci/service/impl/OciBlobServiceImpl.kt | 1 - .../service/impl/OciOperationServiceImpl.kt | 47 ++++++++++++------- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index 396232f9dd..37de94fa4f 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -34,11 +34,15 @@ import com.tencent.bkrepo.oci.pojo.digest.OciDigest object OciLocationUtils { fun buildManifestPath(packageName: String, tag: String): String { - return buildManifestFolderPath(packageName, tag) + OCI_MANIFEST + return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST } - fun buildManifestFolderPath(packageName: String, tag: String): String { - return "/$packageName/manifest/$tag/" + fun buildManifestVersionFolderPath(packageName: String, tag: String): String { + return buildManifestFolderPath(packageName) +"$tag/" + } + + fun buildManifestFolderPath(packageName: String): String { + return "/$packageName/manifest/" } fun buildDigestManifestPathWithReference(packageName: String, reference: String): String { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt index 9a5e57ce73..0f0f3a8ac9 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt @@ -106,7 +106,7 @@ class OciBlobController( } /** - * 删除manifest文件 + * 删除blob文件 * 只能通过digest删除 */ @DeleteMapping(BOLBS_URL) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index 28d7a7b6a1..8180133f5c 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -134,12 +134,13 @@ interface OciOperationService { fun getNodeFullPath(artifactInfo: OciArtifactInfo): String? /** - * 根据sha256值获取对应的node fullpath,md5,size + * 根据sha256值和path获取对应的node fullpath,md5,size */ fun getNodeByDigest( projectId: String, repoName: String, - digestStr: String + digestStr: String, + path: String? = null ): NodeProperty /** diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index d9ffba87a5..136b8ecd7b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -152,7 +152,6 @@ class OciBlobServiceImpl( fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, ociDigest), md5 = nodeProperty.md5!! ) - val repoDetail = repoClient.getRepoDetail(projectId, repoName).data ociOperationService.createNode(nodeCreateRequest) val blobLocation = OciLocationUtils.blobLocation(ociDigest, this) OciResponseUtils.buildBlobMountResponse( diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 752837f514..61ef3e60f1 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -262,7 +262,7 @@ class OciOperationServiceImpl( userId: String, ) { with(artifactInfo) { - val manifestFolder = OciLocationUtils.buildManifestFolderPath(packageName, version) + val manifestFolder = OciLocationUtils.buildManifestVersionFolderPath(packageName, version) val blobsFolder = OciLocationUtils.blobVersionFolderLocation(version, packageName) logger.info("Will delete blobsFolder [$blobsFolder] and manifestFolder $manifestFolder " + "in package $packageName|$version in repo [$projectId/$repoName]") @@ -791,10 +791,12 @@ class OciOperationServiceImpl( if (artifactInfo is OciManifestArtifactInfo) { // 根据类型解析实际存储路径，manifest获取路径有tag/digest if (artifactInfo.isValidDigest) { + val path = OciLocationUtils.buildManifestFolderPath(artifactInfo.packageName) return getNodeByDigest( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, - digestStr = artifactInfo.reference + digestStr = artifactInfo.reference, + path = path ).fullPath } } @@ -807,7 +809,8 @@ class OciOperationServiceImpl( override fun getNodeByDigest( projectId: String, repoName: String, - digestStr: String + digestStr: String, + path: String? ): NodeProperty { val ociDigest = OciDigest(digestStr) val queryModel = NodeQueryBuilder() @@ -815,7 +818,11 @@ class OciOperationServiceImpl( .projectId(projectId) .repoName(repoName) .sha256(ociDigest.getDigestHex()) - .sortByAsc(NODE_FULL_PATH) + .sortByAsc(NODE_FULL_PATH).apply { + path?.let { + this.path(path, OperationType.PREFIX) + } + } val result = nodeClient.search(queryModel.build()).data ?: run { logger.warn( "Could not find $digestStr " + @@ -838,25 +845,29 @@ class OciOperationServiceImpl( * 2 docker-local/nginx/_uploads/ 临时存储上传的blobs，待manifest文件上传成功后移到到对应版本下，如docker-local/nginx/latest */ override fun getDockerNode(artifactInfo: OciArtifactInfo): String? { - if (artifactInfo is OciManifestArtifactInfo) { - // 根据类型解析实际存储路径，manifest获取路径有tag/digest - if (artifactInfo.isValidDigest) + return when (artifactInfo) { + is OciManifestArtifactInfo -> { + // 根据类型解析实际存储路径，manifest获取路径有tag/digest + if (artifactInfo.isValidDigest){ + return getNodeByDigest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + digestStr = artifactInfo.reference, + path = "/${artifactInfo.packageName}/" + ).fullPath + } + return "/${artifactInfo.packageName}/${artifactInfo.reference}/manifest.json" + } + is OciBlobArtifactInfo -> { + val digestStr = artifactInfo.digest ?: StringPool.EMPTY return getNodeByDigest( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, - digestStr = artifactInfo.reference + digestStr = digestStr ).fullPath - return "/${artifactInfo.packageName}/${artifactInfo.reference}/manifest.json" - } - if (artifactInfo is OciBlobArtifactInfo) { - val digestStr = artifactInfo.digest ?: StringPool.EMPTY - return getNodeByDigest( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - digestStr = digestStr - ).fullPath + } + else -> null } - return null } override fun getReturnDomain(request: HttpServletRequest): String { From 97bd531942215501e93986aeb2d948c96f809d6c Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 6 Jul 2023 22:48:25 +0800 Subject: [PATCH 24/41] =?UTF-8?q?feat:=20=E8=8A=82=E7=82=B9=E5=88=B7?= =?UTF-8?q?=E6=96=B0=E6=8E=A5=E5=8F=A3=E9=80=BB=E8=BE=91=E8=B0=83=E6=95=B4?= =?UTF-8?q?=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/pojo/artifact/OciArtifactInfo.kt | 2 +- .../oci/controller/user/UserOciController.kt | 10 ++- .../bkrepo/oci/service/OciOperationService.kt | 6 +- .../service/impl/OciOperationServiceImpl.kt | 63 +++++++++++++++---- 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt index c66265aedf..c1d3a0b401 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt @@ -73,6 +73,6 @@ open class OciArtifactInfo( const val OCI_USER_REPO_SUFFIX = "/repo/{projectId}/{repoName}" const val OCI_USER_TAG_SUFFIX = "/tag/{projectId}/{repoName}/**" const val DOCKER_CATALOG_SUFFIX = "_catalog" - const val OCI_BLOB_NODE_FULLPATH_REFRESH = "/{projectId}/{repoName}/blob/node/refresh" + const val OCI_BLOB_NODE_FULLPATH_REFRESH = "/blob/node/refresh" } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt index 22b7d946c8..0f050ed084 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt @@ -208,12 +208,10 @@ class UserOciController( @PostMapping(OCI_BLOB_NODE_FULLPATH_REFRESH) @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun refreshFullPathOfBlob( - @PathVariable - @ApiParam(value = OCI_PROJECT_ID, required = true) - projectId: String, - @PathVariable - @ApiParam(value = OCI_REPO_NAME, required = true) - repoName: String, + @RequestParam(required = false) + projectId: String? = null, + @RequestParam(required = false) + repoName: String? = null, ): Response { operationService.refreshFullPathOfBlob(projectId, repoName) return ResponseBuilder.success() diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index 8180133f5c..bf03eaa33e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -192,5 +192,9 @@ interface OciOperationService { /** * oci blob 路径调整，由/packageName/blobs/XXX -> /packageName/blobs/version/XXX */ - fun refreshFullPathOfBlob(projectId: String, repoName: String, userId: String = SecurityUtils.getUserId()) + fun refreshFullPathOfBlob( + projectId: String? = null, + repoName: String? =null, + userId: String = SecurityUtils.getUserId() + ) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 61ef3e60f1..8b5a05e251 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -35,6 +35,7 @@ import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.composite.CompositeConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration @@ -95,7 +96,9 @@ import com.tencent.bkrepo.repository.api.MetadataClient import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.api.PackageMetadataClient +import com.tencent.bkrepo.repository.api.ProjectClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest @@ -103,6 +106,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail +import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder import com.tencent.bkrepo.repository.pojo.search.PackageQueryBuilder import com.tencent.devops.plugin.api.PluginManager @@ -131,8 +135,10 @@ class OciOperationServiceImpl( private val storageService: StorageService, private val storageManager: StorageManager, private val repositoryClient: RepositoryClient, + private val projectClient: ProjectClient, private val ociProperties: OciProperties, private val ociReplicationRecordDao: OciReplicationRecordDao, + private val storageCredentialsClient: StorageCredentialsClient, private val pluginManager: PluginManager ) : OciOperationService { @@ -1001,38 +1007,71 @@ class OciOperationServiceImpl( } } - override fun refreshFullPathOfBlob(projectId: String, repoName: String, userId: String) { - val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data + override fun refreshFullPathOfBlob(projectId: String?, repoName: String?, userId: String) { + if (projectId.isNullOrEmpty()) { + projectClient.listProject().data.orEmpty().forEach { + repositoryClient.listRepo( + projectId = projectId!!, type = RepositoryType.OCI.name + ).data.orEmpty().forEach { + refreshPackage(it, userId) + } + repositoryClient.listRepo( + projectId = projectId, type = RepositoryType.DOCKER.name + ).data.orEmpty().forEach { + refreshPackage(it, userId) + } + } + return + } + if (repoName.isNullOrEmpty()) return + val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") - val packageList = packageClient.listAllPackageNames(projectId, repoName).data ?: return - packageList.forEach { pName -> - packageClient.listAllVersion(projectId, repoName, pName).data.orEmpty().forEach { pVersion -> - refreshNode(pName, pVersion, repositoryDetail, userId) + if (repoInfo.type == RepositoryType.OCI || repoInfo.type == RepositoryType.DOCKER){ + refreshPackage(repoInfo, userId) + } + } + + private fun refreshPackage( + repoInfo: RepositoryInfo, + userId: String, + ) { + with(repoInfo) { + val packageList = packageClient.listAllPackageNames(projectId, name).data ?: return + packageList.forEach { pName -> + packageClient.listAllVersion(projectId, name, pName).data.orEmpty().forEach { pVersion -> + refreshNode( + repoInfo = repoInfo, + pName = pName, + pVersion = pVersion, + userId = userId + ) + } } } } private fun refreshNode( + repoInfo: RepositoryInfo, pName: String, pVersion: PackageVersion, - repositoryDetail: RepositoryDetail, userId: String ) { - val packageName = PackageKeys.resolveName(repositoryDetail.type.name.toLowerCase(), pName) + val packageName = PackageKeys.resolveName(repoInfo.type.name.toLowerCase(), pName) val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion.name) logger.info("Manifest $manifestPath will be refreshed") val manifestNode = nodeClient.getNodeDetail( - repositoryDetail.projectId, repositoryDetail.name, manifestPath + repoInfo.projectId, repoInfo.name, manifestPath ).data ?: return + val storageCredentials = repoInfo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } val manifest = loadManifest( - manifestNode.sha256!!, manifestNode.size, repositoryDetail.storageCredentials + manifestNode.sha256!!, manifestNode.size, storageCredentials ) ?: run { logger.warn("The content of manifest.json ${manifestNode.fullPath} is null, check the mediaType.") return } val ociArtifactInfo = OciManifestArtifactInfo( - projectId = repositoryDetail.projectId, - repoName = repositoryDetail.name, + projectId = repoInfo.projectId, + repoName = repoInfo.name, packageName = packageName, version = pVersion.name, reference = pVersion.name, From 90acdad0dd54298045c16464b30b87c658ae7915 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 7 Jul 2023 19:36:40 +0800 Subject: [PATCH 25/41] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pojo/metadata/HelmMaintainerMetadata.kt | 43 ------- .../ResponseProperty.kt} | 50 ++------ .../com/tencent/bkrepo/oci/util/OciUtils.kt | 14 --- .../repository/OciRegistryLocalRepository.kt | 118 +++++++++--------- .../bkrepo/oci/service/OciOperationService.kt | 7 +- .../oci/service/impl/OciBlobServiceImpl.kt | 59 +++++---- .../service/impl/OciOperationServiceImpl.kt | 107 ++++++++-------- .../bkrepo/oci/util/OciResponseUtils.kt | 93 ++++---------- 8 files changed, 178 insertions(+), 313 deletions(-) delete mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt rename src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/{metadata/HelmChartMetadata.kt => response/ResponseProperty.kt} (52%) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt deleted file mode 100644 index 762591e162..0000000000 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.tencent.bkrepo.oci.pojo.metadata - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonInclude - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -data class HelmMaintainerMetadata( - val name: String?, - val email: String?, - val url: String? -) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt similarity index 52% rename from src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt rename to src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt index 011fa99f2b..5715b5c5b3 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt @@ -25,44 +25,16 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.oci.pojo.metadata +package com.tencent.bkrepo.oci.pojo.response -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonInclude -import com.github.zafarkhaja.semver.Version +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.oci.pojo.digest.OciDigest -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -data class HelmChartMetadata( - var apiVersion: String?, - var appVersion: String?, - var created: String?, - var deprecated: Boolean?, - var description: String?, - var digest: String?, - var engine: String?, - var home: String?, - var icon: String?, - var keywords: List = emptyList(), - var maintainers: List = emptyList(), - var name: String, - var sources: List = emptyList(), - var urls: List = emptyList(), - var version: String, - var type: String?, - var annotations: Map? -) : Comparable { - - override fun compareTo(other: HelmChartMetadata): Int { - val result = this.name.compareTo(other.name) - return if (result != 0) { - result - } else { - try { - Version.valueOf(other.version).compareWithBuildsTo(Version.valueOf(this.version)) - } catch (ignored: Exception) { - other.version.compareTo(this.version) - } - } - } -} +data class ResponseProperty( + val digest: OciDigest? = null, + val location: String? = null, + val uuid: String? = null, + val range: Long? = null, + val status: HttpStatus? = null, + val contentLength: Int? = null +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt index 402247b48f..29004f80ff 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt @@ -30,20 +30,15 @@ package com.tencent.bkrepo.oci.util import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.api.util.readJsonString -import com.tencent.bkrepo.common.api.util.readYamlString -import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 -import com.tencent.bkrepo.oci.constant.FILE_EXTENSION import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.model.Descriptor import com.tencent.bkrepo.oci.model.ManifestSchema1 import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.SchemaVersion -import com.tencent.bkrepo.oci.pojo.metadata.HelmChartMetadata -import com.tencent.bkrepo.oci.util.DecompressUtil.getArchivesContent import org.apache.logging.log4j.util.Strings import java.io.InputStream @@ -93,15 +88,6 @@ object OciUtils { } } - fun parseChartInputStream(inputStream: InputStream): HelmChartMetadata { - val result = inputStream.getArchivesContent(FILE_EXTENSION) - return result.byteInputStream().readYamlString() - } - - fun convertToMap(chartInfo: HelmChartMetadata): Map { - return chartInfo.toJsonString().readJsonString() - } - fun manifestIterator(manifest: ManifestSchema2): List { val list = mutableListOf() list.add(manifest.config) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index e2b1d3da05..69fe97e02a 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -63,6 +63,7 @@ import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest import com.tencent.bkrepo.oci.pojo.response.CatalogResponse +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.OciLocationUtils @@ -123,29 +124,13 @@ class OciRegistryLocalRepository( */ override fun onUpload(context: ArtifactUploadContext) { logger.info("Preparing to upload the oci file in repo ${context.artifactInfo.getRepoIdentify()}") - val requestMethod = context.request.method - if (PATCH == requestMethod) { - patchUpload(context) - } else { - val (digest, location) = if (POST == requestMethod) { - postUpload(context) - } else { - putUpload(context) - } - logger.info( - "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded " + - "and will can be accessed in $location" + - " in repo ${context.artifactInfo.getRepoIdentify()}" - ) - if (digest == null || location.isNullOrEmpty()) return - val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) - OciResponseUtils.buildUploadResponse( - domain = domain, - digest = digest, - locationStr = location, - response = context.response - ) - } + val responseProperty = when (context.request.method) { + PATCH -> patchUpload(context) + POST -> postUpload(context) + else -> putUpload(context) + } ?: return + val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) + OciResponseUtils.buildUploadResponse(domain, responseProperty, context.response) } /** @@ -156,27 +141,24 @@ class OciRegistryLocalRepository( * 2:Upload the chunks (PATCH) * 3:Close the session (PUT) */ - private fun patchUpload(context: ArtifactUploadContext) { + private fun patchUpload(context: ArtifactUploadContext): ResponseProperty? { logger.info("Will using patch ways to upload file in repo ${context.artifactInfo.getRepoIdentify()}") - if (context.artifactInfo !is OciBlobArtifactInfo) return + if (context.artifactInfo !is OciBlobArtifactInfo) return null with(context.artifactInfo as OciBlobArtifactInfo) { val range = context.request.getHeader("Content-Range") val length = context.request.contentLength - val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) if (!range.isNullOrEmpty() && length > -1) { logger.info("range $range, length $length, uuid $uuid") val (start, end) = getRangeInfo(range) // 判断要上传的长度是否超长 if (end - start > length - 1) { - OciResponseUtils.buildBlobUploadPatchResponse( - domain = domain, - uuid = uuid!!, - locationStr = OciLocationUtils.blobUUIDLocation(uuid!!, this), - response = HttpContextHolder.getResponse(), + return ResponseProperty( + location = OciLocationUtils.blobUUIDLocation(uuid!!, this), + status = HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE, range = length.toLong(), - status = HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE + uuid = uuid!!, + contentLength = 0 ) - return } } val patchLen = storageService.append( @@ -184,12 +166,12 @@ class OciRegistryLocalRepository( artifactFile = context.getArtifactFile(), storageCredentials = context.repositoryDetail.storageCredentials ) - OciResponseUtils.buildBlobUploadPatchResponse( - domain = domain, + return ResponseProperty( + location = OciLocationUtils.blobUUIDLocation(uuid!!, this), + status = HttpStatus.ACCEPTED, + range = patchLen, uuid = uuid!!, - locationStr = OciLocationUtils.blobUUIDLocation(uuid!!, this), - response = HttpContextHolder.getResponse(), - range = patchLen + contentLength = 0 ) } } @@ -198,7 +180,7 @@ class OciRegistryLocalRepository( * blob 上传，直接使用post * Pushing a blob monolithically ：A single POST request */ - private fun postUpload(context: ArtifactUploadContext): Pair { + private fun postUpload(context: ArtifactUploadContext): ResponseProperty? { val artifactFile = context.getArtifactFile() val digest = OciDigest.fromSha256(artifactFile.getFileSha256()) ociOperationService.storeArtifact( @@ -206,13 +188,19 @@ class OciRegistryLocalRepository( artifactFile = artifactFile, storageCredentials = context.storageCredentials ) + val blobLocation = OciLocationUtils.blobLocation(digest, context.artifactInfo as OciArtifactInfo) logger.info( "Artifact ${context.artifactInfo.getArtifactFullPath()} has " + - "been uploaded to ${context.artifactInfo.getArtifactFullPath()}" + + "been uploaded to ${context.artifactInfo.getArtifactFullPath()}, " + + "and will can be accessed in $blobLocation" + " in repo ${context.artifactInfo.getRepoIdentify()}" ) - val blobLocation = OciLocationUtils.blobLocation(digest, context.artifactInfo as OciArtifactInfo) - return Pair(digest, blobLocation) + return ResponseProperty( + digest = digest, + location = blobLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** @@ -221,14 +209,12 @@ class OciRegistryLocalRepository( * 2 blob POST PATCH with PUT 上传的put模块处理 * 3 manifest PUT上传的逻辑处理 */ - private fun putUpload(context: ArtifactUploadContext): Pair { - if (context.artifactInfo is OciBlobArtifactInfo) { - return putUploadBlob(context) + private fun putUpload(context: ArtifactUploadContext): ResponseProperty? { + return when (context.artifactInfo) { + is OciBlobArtifactInfo -> putUploadBlob(context) + is OciManifestArtifactInfo -> putUploadManifest(context) + else -> null } - if (context.artifactInfo is OciManifestArtifactInfo) { - return putUploadManifest(context) - } - return Pair(null, null) } /** @@ -236,7 +222,7 @@ class OciRegistryLocalRepository( * 1 blob POST with PUT 上传的put模块处理 * 2 blob POST PATCH with PUT 上传的put模块处理 */ - private fun putUploadBlob(context: ArtifactUploadContext): Pair { + private fun putUploadBlob(context: ArtifactUploadContext): ResponseProperty { val artifactInfo = context.artifactInfo as OciBlobArtifactInfo storageService.append( appendId = artifactInfo.uuid!!, @@ -251,19 +237,25 @@ class OciRegistryLocalRepository( storageCredentials = context.storageCredentials, fileInfo = fileInfo ) + val blobLocation = OciLocationUtils.blobLocation(digest, artifactInfo) logger.info( "Artifact ${context.artifactInfo.getArtifactFullPath()} " + "has been uploaded to ${context.artifactInfo.getArtifactFullPath()}" + + "and will can be accessed in $blobLocation" + " in repo ${context.artifactInfo.getRepoIdentify()}" ) - val blobLocation = OciLocationUtils.blobLocation(digest, artifactInfo) - return Pair(digest, blobLocation) + return ResponseProperty( + digest = digest, + location = blobLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** * manifest文件 PUT上传的逻辑处理 */ - private fun putUploadManifest(context: ArtifactUploadContext): Pair { + private fun putUploadManifest(context: ArtifactUploadContext): ResponseProperty { val artifactInfo = context.artifactInfo as OciManifestArtifactInfo val artifactFile = context.getArtifactFile() val digest = OciDigest.fromSha256(artifactFile.getFileSha256()) @@ -272,19 +264,25 @@ class OciRegistryLocalRepository( artifactFile = artifactFile, storageCredentials = context.storageCredentials ) - logger.info( - "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded to ${node!!.fullPath}" + - " in repo ${context.artifactInfo.getRepoIdentify()}" - ) - // 上传manifest文件，同时需要将manifest中对应blob的属性进行补充到blob节点中，同时创建package相关信息 + // 上传manifest文件，同时需要判断manifest中的blob节点是否已经存在，同时创建package相关信息 ociOperationService.updateOciInfo( ociArtifactInfo = artifactInfo, digest = digest, storageCredentials = context.storageCredentials, - nodeDetail = node + nodeDetail = node!! ) val manifestLocation = OciLocationUtils.manifestLocation(digest, artifactInfo) - return Pair(digest, manifestLocation) + logger.info( + "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded to ${node.fullPath}" + + "and will can be accessed in $manifestLocation" + + " in repo ${context.artifactInfo.getRepoIdentify()}" + ) + return ResponseProperty( + digest = digest, + location = manifestLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index bf03eaa33e..2e99f2957a 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -110,11 +110,6 @@ interface OciOperationService { manifestPath: String, ): Boolean - /** - * 当使用追加上传时，文件已存储，只需存储节点信息 - */ - fun createNode(request: NodeCreateRequest): NodeDetail - /** * 保存文件内容(当使用追加上传时，文件已存储，只需存储节点信息) * 特殊：对于manifest文件，node存tag @@ -141,7 +136,7 @@ interface OciOperationService { repoName: String, digestStr: String, path: String? = null - ): NodeProperty + ): NodeProperty? /** * 针对老的docker仓库的数据做兼容性处理 diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index 136b8ecd7b..ac18f57c1b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -48,11 +48,13 @@ import com.tencent.bkrepo.oci.constant.REPO_TYPE import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.pojo.artifact.OciBlobArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import com.tencent.bkrepo.oci.service.OciBlobService import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.ObjectBuildUtils import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciResponseUtils +import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -61,6 +63,7 @@ import org.springframework.stereotype.Service class OciBlobServiceImpl( private val storage: StorageService, private val repoClient: RepositoryClient, + private val nodeClient: NodeClient, private val ociOperationService: OciOperationService, private val permissionManager: PermissionManager ) : OciBlobService { @@ -95,10 +98,15 @@ class OciBlobServiceImpl( logger.info("Will obtain uuid for uploading blobs in repo ${artifactInfo.getRepoIdentify()}.") val uuidCreated = startAppend(this) val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) - OciResponseUtils.buildBlobUploadUUIDResponse( + val responseProperty = ResponseProperty( + uuid = uuidCreated, + location = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + status = HttpStatus.ACCEPTED, + contentLength = 0 + ) + OciResponseUtils.buildUploadResponse( domain, - uuidCreated, - OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + responseProperty, HttpContextHolder.getResponse() ) } else { @@ -120,28 +128,15 @@ class OciBlobServiceImpl( repoName = mountRepoName ) } catch (e: ErrorCodeException) { - val uuidCreated = startAppend(this) - OciResponseUtils.buildBlobMountResponse( - domain = domain, - locationStr = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), - status = HttpStatus.ACCEPTED, - response = HttpContextHolder.getResponse() - ) + buildSessionIdLocationForUpload(this, domain) return } } val nodeProperty = ociOperationService.getNodeByDigest( mountProjectId, mountRepoName, ociDigest.toString() - ) - if (nodeProperty.fullPath == null) { + ) ?: run { logger.warn("Could not find $ociDigest in repo $mountProjectId|$mountRepoName to mount") - val uuidCreated = startAppend(this) - OciResponseUtils.buildBlobMountResponse( - domain = domain, - locationStr = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), - status = HttpStatus.ACCEPTED, - response = HttpContextHolder.getResponse() - ) + buildSessionIdLocationForUpload(this, domain) return } val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( @@ -152,17 +147,35 @@ class OciBlobServiceImpl( fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, ociDigest), md5 = nodeProperty.md5!! ) - ociOperationService.createNode(nodeCreateRequest) + nodeClient.createNode(nodeCreateRequest) val blobLocation = OciLocationUtils.blobLocation(ociDigest, this) - OciResponseUtils.buildBlobMountResponse( + val responseProperty = ResponseProperty( + location = blobLocation, + status = HttpStatus.CREATED + ) + OciResponseUtils.buildUploadResponse( domain = domain, - locationStr = blobLocation, - status = HttpStatus.CREATED, + responseProperty = responseProperty, response = HttpContextHolder.getResponse() ) } } + private fun buildSessionIdLocationForUpload(artifactInfo: OciBlobArtifactInfo, domain: String) { + val uuidCreated = startAppend(artifactInfo) + val responseProperty = ResponseProperty( + uuid = uuidCreated, + location = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + status = HttpStatus.ACCEPTED, + contentLength = 0 + ) + OciResponseUtils.buildUploadResponse( + domain = domain, + responseProperty = responseProperty, + response = HttpContextHolder.getResponse() + ) + } + private fun splitRepoInfo(from: String?): Pair? { if (from.isNullOrEmpty()) return null val values = from.split(CharPool.SLASH) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 8b5a05e251..f68392599b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -69,7 +69,6 @@ import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PROXY_URL import com.tencent.bkrepo.oci.constant.REPO_TYPE import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao -import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.exception.OciFileNotFoundException import com.tencent.bkrepo.oci.exception.OciVersionNotFoundException import com.tencent.bkrepo.oci.extension.ImagePackageInfoPullExtension @@ -100,6 +99,7 @@ import com.tencent.bkrepo.repository.api.ProjectClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest @@ -179,7 +179,11 @@ class OciOperationServiceImpl( metadata = metadata, userId = userId ) - metadataClient.saveMetadata(metadataSaveRequest) + try{ + metadataClient.saveMetadata(metadataSaveRequest) + } catch (ignore: Exception) { + // 并发情况下会出现节点找不到问题 + } } /** @@ -401,14 +405,16 @@ class OciOperationServiceImpl( */ private fun buildNodeCreateRequest( ociArtifactInfo: OciArtifactInfo, - artifactFile: ArtifactFile + artifactFile: ArtifactFile, + metadata: List? = null ): NodeCreateRequest { return ObjectBuildUtils.buildNodeCreateRequest( projectId = ociArtifactInfo.projectId, repoName = ociArtifactInfo.repoName, artifactFile = artifactFile, - fullPath = ociArtifactInfo.getArtifactFullPath() + fullPath = ociArtifactInfo.getArtifactFullPath(), + metadata = metadata ) } @@ -423,42 +429,24 @@ class OciOperationServiceImpl( fileInfo: FileInfo?, proxyUrl: String? ): NodeDetail? { - val request = buildNodeCreateRequest(ociArtifactInfo, artifactFile) + var metadata: List? = null + proxyUrl?.let { + metadata = mutableListOf(MetadataModel(key = PROXY_URL, value = proxyUrl, system = true)) + } + val request = buildNodeCreateRequest(ociArtifactInfo, artifactFile, metadata) val nodeDetail = if (fileInfo != null) { val newNodeRequest = request.copy( size = fileInfo.size, md5 = fileInfo.md5, sha256 = fileInfo.sha256 ) - createNode(newNodeRequest) - null + nodeClient.createNode(newNodeRequest).data } else { storageManager.storeArtifactFile(request, artifactFile, storageCredentials) } - proxyUrl?.let { - saveMetaData( - projectId = request.projectId, - repoName = request.repoName, - fullPath = request.fullPath, - metadata = mutableMapOf(PROXY_URL to proxyUrl), - userId = SecurityUtils.getUserId() - ) - } return nodeDetail } - /** - * 当使用追加上传时，文件已存储，只需存储节点信息 - */ - override fun createNode(request: NodeCreateRequest): NodeDetail { - try { - return nodeClient.createNode(request).data!! - } catch (exception: Exception) { - // 异常往上抛 - throw exception - } - } - /** * 更新整个blob相关信息,blob相关的mediatype，version等信息需要从manifest中获取 */ @@ -516,15 +504,15 @@ class OciOperationServiceImpl( size: Long, storageCredentials: StorageCredentials? ): ManifestSchema2? { - val manifestBytes = storageService.load( - sha256, - Range.full(size), - storageCredentials - )!!.readText() - return try { + val manifestBytes = storageService.load( + sha256, + Range.full(size), + storageCredentials + )!!.readText() + OciUtils.stringToManifestV2(manifestBytes) - } catch (e: OciBadRequestException) { + } catch (e: Exception) { null } } @@ -652,7 +640,7 @@ class OciOperationServiceImpl( existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, userId) if (!existFlag) { // 第三方同步场景下，如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 - return Pair(false, 0) + return Pair(existFlag, 0) } } return Pair(existFlag, size) @@ -699,14 +687,13 @@ class OciOperationServiceImpl( // 并发情况下，版本目录下可能存在着非该版本的blob // 覆盖上传时会先删除原有目录，并发情况下可能导致blobs节点不存在 var nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) - if (nodeProperty.fullPath == null) { + if (nodeProperty == null) { nodeProperty = nodeClient.getDeletedNodeDetail( projectId, repoName, fullPath ).data?.firstOrNull{ it.sha256 == descriptor.sha256 }?.let { NodeProperty(it.fullPath, it.md5, it.size) } ?: return false } - if (nodeProperty.fullPath.isNullOrEmpty()) return false val newPath = OciLocationUtils.blobVersionPathLocation(reference, packageName, descriptor.filename) if (newPath != nodeProperty.fullPath) { // 创建新路径节点 /packageName/blobs/version/xxx @@ -719,7 +706,7 @@ class OciOperationServiceImpl( md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, userId = userId ) - createNode(nodeCreateRequest) + nodeClient.createNode(nodeCreateRequest) } // 删除临时存储路径节点 /packageName/blobs/xxx deleteNode(projectId, repoName, fullPath, userId) @@ -785,7 +772,10 @@ class OciOperationServiceImpl( metadata = metadata, userId = SecurityUtils.getUserId() ) - packageMetadataClient.saveMetadata(metadataSaveRequest) + try { + packageMetadataClient.saveMetadata(metadataSaveRequest) + } catch (ignore: Exception) { + } } @@ -794,17 +784,16 @@ class OciOperationServiceImpl( * 特殊：manifest文件按tag存储， 但是查询时存在tag/digest */ override fun getNodeFullPath(artifactInfo: OciArtifactInfo): String? { - if (artifactInfo is OciManifestArtifactInfo) { - // 根据类型解析实际存储路径，manifest获取路径有tag/digest - if (artifactInfo.isValidDigest) { - val path = OciLocationUtils.buildManifestFolderPath(artifactInfo.packageName) - return getNodeByDigest( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - digestStr = artifactInfo.reference, - path = path - ).fullPath - } + if (artifactInfo is OciManifestArtifactInfo && artifactInfo.isValidDigest) { + // 根据类型解析实际存储路径，manifest获取路径有tag/digest, + // 当为digest 时，查当前仓库下，在package目录下，且sha256为digest的节点 + val path = OciLocationUtils.buildManifestFolderPath(artifactInfo.packageName) + return getNodeByDigest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + digestStr = artifactInfo.reference, + path = path + )?.fullPath } return artifactInfo.getArtifactFullPath() } @@ -817,7 +806,7 @@ class OciOperationServiceImpl( repoName: String, digestStr: String, path: String? - ): NodeProperty { + ): NodeProperty? { val ociDigest = OciDigest(digestStr) val queryModel = NodeQueryBuilder() .select(NODE_FULL_PATH, MD5, OCI_NODE_SIZE) @@ -829,14 +818,14 @@ class OciOperationServiceImpl( this.path(path, OperationType.PREFIX) } } - val result = nodeClient.search(queryModel.build()).data ?: run { + val result = nodeClient.search(queryModel.build()).data + if (result == null || result.records.isEmpty()) { logger.warn( "Could not find $digestStr " + "in repo $projectId|$repoName" ) - return NodeProperty() + return null } - if (result.records.isEmpty()) return NodeProperty() return NodeProperty( fullPath = result.records[0][NODE_FULL_PATH] as String, md5 = result.records[0][MD5] as String?, @@ -860,7 +849,7 @@ class OciOperationServiceImpl( repoName = artifactInfo.repoName, digestStr = artifactInfo.reference, path = "/${artifactInfo.packageName}/" - ).fullPath + )?.fullPath } return "/${artifactInfo.packageName}/${artifactInfo.reference}/manifest.json" } @@ -870,7 +859,7 @@ class OciOperationServiceImpl( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, digestStr = digestStr - ).fullPath + )?.fullPath } else -> null } @@ -1050,6 +1039,10 @@ class OciOperationServiceImpl( } } + /** + * 调整blob目录 + * 从/packageName/blobs/xxx 到/packageName/blobs/version/xxx + */ private fun refreshNode( repoInfo: RepositoryInfo, pName: String, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt index 674c1888c8..6cca3c70a4 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt @@ -49,6 +49,7 @@ import com.tencent.bkrepo.oci.constant.HTTP_PROTOCOL_HTTP import com.tencent.bkrepo.oci.constant.HTTP_PROTOCOL_HTTPS import com.tencent.bkrepo.oci.constant.OCI_API_PREFIX import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import org.springframework.http.HttpHeaders import java.io.UnsupportedEncodingException import java.net.URI @@ -99,58 +100,11 @@ object OciResponseUtils { ) } - fun buildUploadResponse(domain: String, digest: OciDigest, locationStr: String, response: HttpServletResponse) { + fun buildUploadResponse(domain: String, responseProperty: ResponseProperty, response: HttpServletResponse) { uploadResponse( domain = domain, response = response, - status = HttpStatus.CREATED, - locationStr = locationStr, - digest = digest.toString(), - contentLength = 0 - ) - } - - fun buildBlobUploadUUIDResponse(domain: String, uuid: String, locationStr: String, response: HttpServletResponse) { - uploadResponse( - domain = domain, - response = response, - status = HttpStatus.ACCEPTED, - locationStr = locationStr, - uuid = uuid, - contentLength = 0 - ) - } - - fun buildBlobUploadPatchResponse( - domain: String, - uuid: String, - locationStr: String, - status: HttpStatus = HttpStatus.ACCEPTED, - response: HttpServletResponse, - range: Long - ) { - uploadResponse( - domain = domain, - response = response, - status = status, - locationStr = locationStr, - uuid = uuid, - contentLength = 0, - range = range - ) - } - - fun buildBlobMountResponse( - domain: String, - locationStr: String, - status: HttpStatus = HttpStatus.ACCEPTED, - response: HttpServletResponse - ) { - uploadResponse( - domain = domain, - response = response, - status = status, - locationStr = locationStr + responseProperty = responseProperty ) } @@ -175,29 +129,26 @@ object OciResponseUtils { private fun uploadResponse( domain: String, response: HttpServletResponse = HttpContextHolder.getResponse(), - status: HttpStatus, - locationStr: String, - digest: String? = null, - uuid: String? = null, - range: Long? = null, - contentLength: Int? = null + responseProperty: ResponseProperty ) { - val location = getResponseLocationURI(locationStr, domain) - response.status = status.value - response.addHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION) - digest?.let { - response.addHeader(DOCKER_CONTENT_DIGEST, digest) - } - response.addHeader(HttpHeaders.LOCATION, location) - uuid?.let { - response.addHeader(BLOB_UPLOAD_SESSION_ID, uuid) - response.addHeader(DOCKER_UPLOAD_UUID, uuid) - } - contentLength?.let { - response.addHeader(CONTENT_LENGTH, contentLength.toString()) - } - range?.let { - response.addHeader(RANGE, "0-${range - 1}") + with(responseProperty) { + val location = getResponseLocationURI(location!!, domain) + response.status = status!!.value + response.addHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION) + digest?.let { + response.addHeader(DOCKER_CONTENT_DIGEST, digest.toString()) + } + response.addHeader(HttpHeaders.LOCATION, location) + uuid?.let { + response.addHeader(BLOB_UPLOAD_SESSION_ID, uuid) + response.addHeader(DOCKER_UPLOAD_UUID, uuid) + } + contentLength?.let { + response.addHeader(CONTENT_LENGTH, contentLength.toString()) + } + range?.let { + response.addHeader(RANGE, "0-${range!! - 1}") + } } } From 1c54a693357924d3a4b593142a64aab8580678d7 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 7 Jul 2023 19:39:14 +0800 Subject: [PATCH 26/41] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index 2e99f2957a..a04292eab7 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -40,7 +40,6 @@ import com.tencent.bkrepo.oci.pojo.response.OciImageResult import com.tencent.bkrepo.oci.pojo.response.OciTagResult import com.tencent.bkrepo.oci.pojo.user.PackageVersionInfo import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import javax.servlet.http.HttpServletRequest interface OciOperationService { From 9b2ac31912485877a2af96a252acd7c26f0db0b7 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 7 Jul 2023 19:46:41 +0800 Subject: [PATCH 27/41] =?UTF-8?q?feat:=20=E5=8E=BB=E6=8E=89=E6=B3=A8?= =?UTF-8?q?=E9=87=8A=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/artifact/repository/OciRegistryRemoteRepository.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index f4005cccc6..4f6a032cab 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -386,9 +386,7 @@ class OciRegistryRemoteRepository( } } - /** - * 可能请求的地址一样，但是 - */ + private fun getScope(remoteUrl: String, imageName: String): String { val baseUrl = URL(remoteUrl) val target = baseUrl.path.removePrefix(StringPool.SLASH) From 242cc7293b09bb53eef5313bd9d4d78e65c22f66 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 7 Jul 2023 20:03:13 +0800 Subject: [PATCH 28/41] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/listener/base/EventExecutor.kt | 2 +- .../bkrepo/oci/service/OciOperationService.kt | 21 -------- .../service/impl/OciOperationServiceImpl.kt | 53 +++---------------- 3 files changed, 9 insertions(+), 67 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt index 0d822c99a8..c8bcdf504e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt @@ -75,7 +75,7 @@ class EventExecutor( EventType.REPO_CREATED, EventType.REPO_REFRESHED, EventType.REPO_UPDATED -> { ociOperationService.getPackagesFromThirdPartyRepo(event.projectId, event.repoName) } - else -> UnsupportedOperationException() + else -> throw UnsupportedOperationException() } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index a04292eab7..782db9b951 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -44,27 +44,6 @@ import javax.servlet.http.HttpServletRequest interface OciOperationService { - /** - * 保存节点元数据 - */ - fun saveMetaData( - projectId: String, - repoName: String, - fullPath: String, - metadata: MutableMap, - userId: String - ) - - /** - * 需要将blob中相关metadata写进package version中 - */ - fun updatePackageInfo( - ociArtifactInfo: OciArtifactInfo, - packageKey: String, - appVersion: String? = null, - description: String? = null - ) - /** * 查询包版本详情 */ diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index f68392599b..daf6bc2150 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -162,50 +162,6 @@ class OciOperationServiceImpl( ) } - /** - * 保存节点元数据 - */ - override fun saveMetaData( - projectId: String, - repoName: String, - fullPath: String, - metadata: MutableMap, - userId: String - ) { - val metadataSaveRequest = ObjectBuildUtils.buildMetadataSaveRequest( - projectId = projectId, - repoName = repoName, - fullPath = fullPath, - metadata = metadata, - userId = userId - ) - try{ - metadataClient.saveMetadata(metadataSaveRequest) - } catch (ignore: Exception) { - // 并发情况下会出现节点找不到问题 - } - } - - /** - * 需要将blob中相关metadata写进package version中 - */ - override fun updatePackageInfo( - ociArtifactInfo: OciArtifactInfo, - packageKey: String, - appVersion: String?, - description: String? - ) { - with(ociArtifactInfo) { - val packageUpdateRequest = ObjectBuildUtils.buildPackageUpdateRequest( - artifactInfo = this, - name = packageName, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - packageClient.updatePackage(packageUpdateRequest) - } - } /** * 删除package @@ -556,16 +512,23 @@ class OciOperationServiceImpl( digestList = digestList, sourceType = sourceType ) - saveMetaData( + + val metadataSaveRequest = ObjectBuildUtils.buildMetadataSaveRequest( projectId = projectId, repoName = repoName, fullPath = fullPath, metadata = metadata, userId = SecurityUtils.getUserId() ) + try{ + metadataClient.saveMetadata(metadataSaveRequest) + } catch (ignore: Exception) { + // 并发情况下会出现节点找不到问题 + } } + /** * 同步blob层的数据和config里面的数据 */ From 5b426892e884210ff2bf6c115b7d87cc7c81dc32 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 7 Jul 2023 20:16:10 +0800 Subject: [PATCH 29/41] =?UTF-8?q?feat:=20=E7=A6=81=E6=AD=A2v1=E7=9A=84?= =?UTF-8?q?=E9=95=9C=E5=83=8Fmanifest=E6=96=87=E4=BB=B6=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=EF=BC=8C=E9=83=A8=E5=88=86=E5=AE=A2=E6=88=B7=E7=AB=AF=E6=8B=89?= =?UTF-8?q?=E5=8F=96=E4=BC=9A=E6=8A=A5=E9=94=99=20#908?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/constant/OciMessageCode.kt | 1 + .../service/impl/OciOperationServiceImpl.kt | 28 ++++++++----------- .../resources/i18n/messages_en.properties | 1 + .../resources/i18n/messages_zh_CN.properties | 1 + .../resources/i18n/messages_zh_TW.properties | 1 + 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt index cc8226539b..a4ec52a40b 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt @@ -37,6 +37,7 @@ enum class OciMessageCode(private val key: String) : MessageCode { OCI_DELETE_RULES("oci.delete.rules"), OCI_VERSION_NOT_FOUND("oci.version.not.found"), OCI_MANIFEST_INVALID("oci.manifest.invalid"), + OCI_MANIFEST_SCHEMA1_NOT_SUPPORT("oci.manifest.schema1.not.support"), OCI_REMOTE_CONFIGURATION_ERROR("oci.remote.configuration.error"), OCI_REMOTE_CREDENTIALS_INVALID("oci.remote.credentials.invalid") ; diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index daf6bc2150..dd6782696e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -54,7 +54,6 @@ import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties import com.tencent.bkrepo.oci.constant.DESCRIPTION -import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 import com.tencent.bkrepo.oci.constant.DOWNLOADS import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_BY import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_DATE @@ -69,6 +68,7 @@ import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PROXY_URL import com.tencent.bkrepo.oci.constant.REPO_TYPE import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao +import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.exception.OciFileNotFoundException import com.tencent.bkrepo.oci.exception.OciVersionNotFoundException import com.tencent.bkrepo.oci.extension.ImagePackageInfoPullExtension @@ -417,20 +417,18 @@ class OciOperationServiceImpl( "Will start to update oci info for ${ociArtifactInfo.getArtifactFullPath()} " + "in repo ${ociArtifactInfo.getRepoIdentify()}" ) - + // https://github.com/docker/docker-ce/blob/master/components/engine/distribution/push_v2.go + // docker 客户端上传manifest时先按照schema2的格式上传， + // 如失败则按照schema1格式上传，但是非docker客户端不兼容schema1版本manifest val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) - // 将该版本对应的blob sha256放到manifest节点的元数据中 - val (mediaType, digestList) = if (manifest == null) { - Pair(DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1, null) + ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) + // 更新manifest文件的metadata + val mediaType = if (manifest.mediaType.isNullOrEmpty()) { + HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE } else { - // 更新manifest文件的metadata - val mediaTypeV2 = if (manifest.mediaType.isNullOrEmpty()) { - HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE - } else { - manifest.mediaType - } - Pair(mediaTypeV2, OciUtils.manifestIteratorDigest(manifest)) + manifest.mediaType } + val digestList = OciUtils.manifestIteratorDigest(manifest) // 更新manifest节点元数据 updateNodeMetaData( @@ -536,7 +534,7 @@ class OciOperationServiceImpl( ociArtifactInfo: OciManifestArtifactInfo, nodeDetail: NodeDetail, sourceType: ArtifactChannel? = null, - manifest: ManifestSchema2? = null, + manifest: ManifestSchema2, userId: String = SecurityUtils.getUserId() ): Boolean { logger.info( @@ -579,12 +577,10 @@ class OciOperationServiceImpl( * 针对v2版本manifest文件做特殊处理 */ private fun manifestHandler( - manifest: ManifestSchema2?, + manifest: ManifestSchema2, ociArtifactInfo: OciManifestArtifactInfo, userId: String = SecurityUtils.getUserId() ): Pair { - //当manifest为空时，可能是v1版本镜像,直接返回 - if (manifest == null) return Pair(true, 0) // 用于判断是否所有blob都以存在 var existFlag = true // 统计所有mainfest中的文件size作为整个package version的size diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties index 0749d8bb5a..ae019aecf7 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties @@ -36,5 +36,6 @@ oci.repo.not.found=Oci repository [{0}] not found! oci.delete.rules=Delete the blob [{0}] identified by digest! oci.version.not.found=The version [{0}] of artifact not found in repo [{1}] ! oci.manifest.invalid=Manifest invalid! +oci.manifest.schema1.not.support=Schema1 of manifest not supported! oci.remote.configuration.error= Remote configuration error, error is [{0}]! oci.remote.credentials.invalid= Remote credentials invalid, error is [{0}]! \ No newline at end of file diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties index 14f13c387c..821372ca6b 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties @@ -36,5 +36,6 @@ oci.repo.not.found=Oci仓库 [{0}] 不存在！ oci.delete.rules=制品 [{0}] 只能使用digest删除！ oci.version.not.found=制品版本 [{0}] 在仓库 [{1}] 中不存在！ oci.manifest.invalid=请检查Manifest内容是否符合规范！ +oci.manifest.schema1.not.support=Manifest文件不支持Schema1版本! oci.remote.configuration.error= 请检查远程仓库配置信息, 错误：[{0}]! oci.remote.credentials.invalid= 代理鉴权信息有误: [{0}]，请确认！ \ No newline at end of file diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties index 80014534de..bec9c7f4eb 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties @@ -36,5 +36,6 @@ oci.repo.not.found=Oci倉庫 [{0}] 不存在！ oci.delete.rules=製品 [{0}] 只能digest刪除！ oci.version.not.found=製品版本 [{0}] 在倉庫 [{1}] 中不存在！ oci.manifest.invalid=請檢查Manifest內容是否符合規範！ +oci.manifest.schema1.not.support=Manifest文件不支持Schema1版本！ oci.remote.configuration.error= 請檢查遠程倉庫配置信息, 錯誤：[{0}]! oci.remote.credentials.invalid= 代理鑒權信息有誤: [{0}] ，請確認！ \ No newline at end of file From 24b25b4c083b3a9deaa498eee8802fd6122025c0 Mon Sep 17 00:00:00 2001 From: zacYL Date: Tue, 4 Jul 2023 22:17:08 +0800 Subject: [PATCH 30/41] =?UTF-8?q?bug:=20feign=E8=AF=B7=E6=B1=82=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E8=BF=94=E5=9B=9E=E5=85=88=E8=BD=AC=E6=8D=A2=E6=88=90?= =?UTF-8?q?String=E5=86=8D=E8=BF=9B=E8=A1=8C=E7=B1=BB=E5=9E=8B=E8=BD=AC?= =?UTF-8?q?=E6=8D=A2=20#892?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit ec4614784cb5bb0b8b2149dfec094cd53df93c37) --- .../kotlin/com/tencent/bkrepo/oci/pojo/node/NodeProperty.kt | 2 +- .../com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt | 2 +- .../tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/node/NodeProperty.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/node/NodeProperty.kt index b9db729baf..1ab80a577d 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/node/NodeProperty.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/node/NodeProperty.kt @@ -30,5 +30,5 @@ package com.tencent.bkrepo.oci.pojo.node data class NodeProperty( var fullPath: String? = null, var md5: String? = null, - var size: Int? = null, + var size: Long? = null, ) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index ac18f57c1b..df449104c7 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -142,7 +142,7 @@ class OciBlobServiceImpl( val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( projectId = projectId, repoName = repoName, - size = nodeProperty.size!!.toLong(), + size = nodeProperty.size!!, sha256 = ociDigest.hex, fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, ociDigest), md5 = nodeProperty.md5!! diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index dd6782696e..4c3c5c2b74 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -788,7 +788,7 @@ class OciOperationServiceImpl( return NodeProperty( fullPath = result.records[0][NODE_FULL_PATH] as String, md5 = result.records[0][MD5] as String?, - size = result.records[0][OCI_NODE_SIZE] as Int? + size = result.records[0][OCI_NODE_SIZE].toString().toLong() ) } From 707863b90e31d4cfc94fdab1b62383b16f8f66cc Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 12 Jul 2023 18:14:47 +0800 Subject: [PATCH 31/41] =?UTF-8?q?bug:=20catalog=E7=9B=B8=E5=85=B3=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E8=B0=83=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/constant/OciConstants.kt | 3 +- .../oci/pojo/artifact/OciArtifactInfo.kt | 7 +++-- .../repository/OciRegistryRemoteRepository.kt | 28 +++++++++++++++---- .../resolver/OciTagArtifactInfoResolver.kt | 13 +++------ .../oci/controller/user/CatalogController.kt | 22 ++++++--------- .../oci/controller/user/OciTagController.kt | 10 ++----- 6 files changed, 45 insertions(+), 38 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index 694832db1e..59f7000686 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -95,7 +95,8 @@ const val MD5 = "md5" const val OCI_MANIFEST = "manifest.json" const val STAGE_TAG = "stageTag" -const val REQUEST_TAG_LIST = "tagList" +const val TAG_LIST_REQUEST = "tagList" +const val CATALOG_REQUEST = "catalog" const val REQUEST_IMAGE = "image" // OCIScheme is the URL scheme for OCI-based requests diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt index c1d3a0b401..ad86c29317 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt @@ -60,11 +60,14 @@ open class OciArtifactInfo( const val BOLBS_UPLOAD_SECOND_STEP_URL = "/v2/{projectId}/{repoName}/**/blobs/uploads/{uuid}" // tags get - const val TAGS_URL = "/v2/{projectId}/{repoName}/**/tags/list" + const val TAGS_LIST_SUFFIX = "/tags/list" + // Retrieve a sorted, json list of repositories available in the registry. + const val DOCKER_CATALOG_SUFFIX = "/v2/_catalog" // version详情获取 const val OCI_VERSION_DETAIL = "/version/detail/{projectId}/{repoName}" + // 额外的package或者version 删除接口 const val OCI_PACKAGE_DELETE_URL = "/package/delete/{projectId}/{repoName}" const val OCI_VERSION_DELETE_URL = "/version/delete/{projectId}/{repoName}" @@ -72,7 +75,7 @@ open class OciArtifactInfo( const val OCI_USER_LAYER_SUFFIX = "/layer/{projectId}/{repoName}/**/{id}" const val OCI_USER_REPO_SUFFIX = "/repo/{projectId}/{repoName}" const val OCI_USER_TAG_SUFFIX = "/tag/{projectId}/{repoName}/**" - const val DOCKER_CATALOG_SUFFIX = "_catalog" + const val OCI_BLOB_NODE_FULLPATH_REFRESH = "/blob/node/refresh" } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 4f6a032cab..6aa2b43cc5 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -57,6 +57,7 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.artifactStream import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter +import com.tencent.bkrepo.oci.constant.CATALOG_REQUEST import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_V2 import com.tencent.bkrepo.oci.constant.DOCKER_LINK import com.tencent.bkrepo.oci.constant.LAST_TAG @@ -67,10 +68,11 @@ import com.tencent.bkrepo.oci.constant.OCI_FILTER_ENDPOINT import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PROXY_URL -import com.tencent.bkrepo.oci.constant.REQUEST_TAG_LIST +import com.tencent.bkrepo.oci.constant.TAG_LIST_REQUEST import com.tencent.bkrepo.oci.exception.OciForbiddenRequestException import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciBlobArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo @@ -257,7 +259,8 @@ class OciRegistryRemoteRepository( fun createRemoteDownloadUrl(context: ArtifactContext, property: RemoteRequestProperty): String { return when (property.type) { - REQUEST_TAG_LIST -> createCatalogUrl(property) + CATALOG_REQUEST -> createCatalogUrl(property) + TAG_LIST_REQUEST -> createTagListUrl(property) else -> createUrl(property) } } @@ -285,7 +288,7 @@ class OciRegistryRemoteRepository( RemoteRequestProperty( url = url, params = params, - type = REQUEST_TAG_LIST, + type = CATALOG_REQUEST, imageName = StringPool.EMPTY ) } else { @@ -294,6 +297,7 @@ class OciRegistryRemoteRepository( url = url, fullPath = fullPath, params = params, + type = TAG_LIST_REQUEST, imageName = artifactInfo.packageName ) } @@ -326,15 +330,27 @@ class OciRegistryRemoteRepository( */ private fun createCatalogUrl(property: RemoteRequestProperty): String { with(property) { - val baseUrl = URL(url) - val builder = UriBuilder.fromPath(OCI_API_PREFIX) - .host(baseUrl.host).scheme(baseUrl.protocol) + val builder = UriBuilder.fromUri(url) .path(DOCKER_CATALOG_SUFFIX) .queryParam(params) return builder.build().toString() } } + /** + * 拼接tag list url + */ + private fun createTagListUrl(property: RemoteRequestProperty): String { + with(property) { + val builder = UriBuilder.fromUri(url) + .path(OCI_API_PREFIX) + .path(imageName) + .path(TAGS_LIST_SUFFIX) + .queryParam(params) + return builder.build().toString() + } + } + private fun getAuthenticationCode( proxyUrl: String, wwwAuthenticate: String?, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt index 0dbd8a7f01..d5830211a3 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt @@ -38,13 +38,12 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.resolve.path.ArtifactInfoResolver import com.tencent.bkrepo.common.artifact.resolve.path.Resolver import com.tencent.bkrepo.oci.constant.OCI_TAG -import com.tencent.bkrepo.oci.constant.USER_API_PREFIX import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo -import io.undertow.servlet.spec.HttpServletRequestImpl -import javax.servlet.http.HttpServletRequest import org.springframework.stereotype.Component import org.springframework.web.servlet.HandlerMapping +import javax.servlet.http.HttpServletRequest @Component @Resolver(OciTagArtifactInfo::class) @@ -58,17 +57,14 @@ class OciTagArtifactInfoResolver : ArtifactInfoResolver { ): ArtifactInfo { val requestURL = ArtifactContextHolder.getUrlPath(this.javaClass.name)!! return when { - requestURL.contains(TAG_PREFIX) -> { + requestURL.contains(TAGS_LIST_SUFFIX) -> { val requestUrl = request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE).toString() - val packageName = requestUrl.removePrefix("$USER_API_PREFIX/tag/$projectId/$repoName/") + val packageName = requestUrl.removePrefix("/$projectId/$repoName/v2/").removeSuffix(TAGS_LIST_SUFFIX) validate(packageName) val tag = request.getParameter(OCI_TAG) ?: StringPool.EMPTY OciTagArtifactInfo(projectId, repoName, packageName, tag) } requestURL.contains(DOCKER_CATALOG_SUFFIX) -> { - val params = (request as HttpServletRequestImpl).queryParameters - val projectId = params?.get("projectId")?.first ?: StringPool.EMPTY - val repoName = params?.get("repoName")?.first ?: StringPool.EMPTY OciTagArtifactInfo(projectId, repoName, StringPool.EMPTY, StringPool.EMPTY) } else -> { @@ -89,6 +85,5 @@ class OciTagArtifactInfoResolver : ArtifactInfoResolver { companion object { const val PACKAGE_NAME_PATTERN = "[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*" - const val TAG_PREFIX = "/ext/tag/" } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt index b7b8a0f4cc..d70aa77742 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt @@ -34,11 +34,11 @@ package com.tencent.bkrepo.oci.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.security.permission.Permission -import com.tencent.bkrepo.oci.config.OciProperties import com.tencent.bkrepo.oci.constant.DOCKER_API_VERSION import com.tencent.bkrepo.oci.constant.DOCKER_HEADER_API_VERSION import com.tencent.bkrepo.oci.constant.DOCKER_LINK -import com.tencent.bkrepo.oci.constant.OCI_FILTER_ENDPOINT +import com.tencent.bkrepo.oci.constant.OCI_PROJECT_ID +import com.tencent.bkrepo.oci.constant.OCI_REPO_NAME import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.service.OciCatalogService @@ -47,29 +47,25 @@ import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController @RestController -@RequestMapping(OCI_FILTER_ENDPOINT) -class CatalogController( - private val catalogService: OciCatalogService, - private val ociProperties: OciProperties -) { +class CatalogController(private val catalogService: OciCatalogService) { /** * 返回仓库下所有image名列表 */ - @GetMapping(DOCKER_CATALOG_SUFFIX) + @GetMapping("/{projectId}/{repoName}$DOCKER_CATALOG_SUFFIX") @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun list( artifactInfo: OciTagArtifactInfo, - @RequestParam(required = true) - @ApiParam(required = true) + @PathVariable + @ApiParam(value = OCI_PROJECT_ID, required = true) projectId: String, - @RequestParam(required = true) - @ApiParam(required = true) + @PathVariable + @ApiParam(value = OCI_REPO_NAME, required = true) repoName: String, @RequestParam(required = false) @ApiParam(value = "n", required = false) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt index e4c265832b..c005855fdc 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt @@ -30,9 +30,8 @@ package com.tencent.bkrepo.oci.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.security.permission.Permission -import com.tencent.bkrepo.oci.config.OciProperties import com.tencent.bkrepo.oci.constant.DOCKER_LINK -import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_URL +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciTagService @@ -48,15 +47,12 @@ import org.springframework.web.bind.annotation.RestController * oci tag controller */ @RestController -class OciTagController( - private val ociTagService: OciTagService, - private val ociProperties: OciProperties -) { +class OciTagController(private val ociTagService: OciTagService) { /** * 获取blob对应的tag信息 */ @Permission(type = ResourceType.REPO, action = PermissionAction.READ) - @RequestMapping(TAGS_URL, method = [RequestMethod.GET]) + @RequestMapping("/{projectId}/{repoName}/**$TAGS_LIST_SUFFIX", method = [RequestMethod.GET]) fun getTagList( artifactInfo: OciTagArtifactInfo, @RequestParam n: Int?, From ed035986743993f1e491425be1da9d2fd65f1491 Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 13 Jul 2023 11:10:10 +0800 Subject: [PATCH 32/41] =?UTF-8?q?bug:=20=E8=B7=AF=E5=BE=84=E6=8B=BC?= =?UTF-8?q?=E6=8E=A5=E8=B0=83=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/OciRegistryRemoteRepository.kt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 6aa2b43cc5..8a6833e65b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -330,10 +330,7 @@ class OciRegistryRemoteRepository( */ private fun createCatalogUrl(property: RemoteRequestProperty): String { with(property) { - val builder = UriBuilder.fromUri(url) - .path(DOCKER_CATALOG_SUFFIX) - .queryParam(params) - return builder.build().toString() + return UrlFormatter.buildUrl(url, DOCKER_CATALOG_SUFFIX, params) } } @@ -342,12 +339,12 @@ class OciRegistryRemoteRepository( */ private fun createTagListUrl(property: RemoteRequestProperty): String { with(property) { - val builder = UriBuilder.fromUri(url) + val url = UriBuilder.fromUri(url) .path(OCI_API_PREFIX) .path(imageName) .path(TAGS_LIST_SUFFIX) - .queryParam(params) - return builder.build().toString() + .build().toString() + return UrlFormatter.addParams(url, params) } } From a4f1b5f6d63de9098060bf41a3a29b76f881c9b9 Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 13 Jul 2023 11:34:52 +0800 Subject: [PATCH 33/41] =?UTF-8?q?bug:=20=E5=BD=93=E5=8C=85=E4=B8=8B?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E7=89=88=E6=9C=AC=E6=97=B6=E9=9C=80=E8=A6=81?= =?UTF-8?q?=E5=88=A0=E6=8E=89=E5=8C=85=E7=9B=AE=E5=BD=95=EF=BC=8C=E6=B8=85?= =?UTF-8?q?=E9=99=A4=E5=AD=98=E5=82=A8=E5=9C=A8blobs=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E4=B8=8B=E6=95=B0=E6=8D=AE=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/service/impl/OciOperationServiceImpl.kt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 4c3c5c2b74..2c1c2d287c 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -173,10 +173,10 @@ class OciOperationServiceImpl( val packageKey = PackageKeys.ofName(repoDetail.type.name.toLowerCase(), packageName) if (version.isNotBlank()) { packageClient.findVersionByName( - projectId, - repoName, - packageKey, - version + projectId = projectId, + repoName = repoName, + packageKey = packageKey, + version = version ).data?.let { removeVersion( artifactInfo = this, @@ -185,6 +185,11 @@ class OciOperationServiceImpl( packageKey = packageKey ) } ?: throw VersionNotFoundException(version) + if (!packageClient.listAllVersion(projectId, repoName, packageKey).data.isNullOrEmpty()) { + return + } + // 当没有版本时删除删除package目录 + deleteNode(projectId, repoName, "${StringPool.SLASH}$packageName", userId) } else { // 删除package目录 deleteNode(projectId, repoName, "${StringPool.SLASH}$packageName", userId) From 12342768161e5a6f221d0caa1973e741e480a3ff Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Tue, 8 Aug 2023 16:47:51 +0800 Subject: [PATCH 34/41] =?UTF-8?q?feat:=20=E5=8E=86=E5=8F=B2=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=BF=AE=E6=AD=A3=E6=96=B9=E6=A1=88=E8=B0=83=E6=95=B4?= =?UTF-8?q?=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/job/batch/OciBlobNodeRefreshJob.kt | 138 ++++++++++++++++++ .../OciBlobNodeRefreshJobProperties.kt | 40 +++++ .../com/tencent/bkrepo/oci/api/OciClient.kt | 19 +++ .../bkrepo/oci/constant/OciConstants.kt | 2 + .../service/OciPackageController.kt | 15 ++ .../oci/controller/user/UserOciController.kt | 15 -- .../bkrepo/oci/service/OciOperationService.kt | 19 ++- .../service/impl/OciOperationServiceImpl.kt | 117 +++++++-------- .../bkrepo/oci/util/ObjectBuildUtils.kt | 4 +- 9 files changed, 293 insertions(+), 76 deletions(-) create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt new file mode 100644 index 0000000000..b2a3437e16 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt @@ -0,0 +1,138 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch + +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.job.TYPE +import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.config.properties.OciBlobNodeRefreshJobProperties +import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.find +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +/** + * 用于将存储在blobs目录下的公共blob节点全部迁移到对应版本目录下， + * 即从/{packageName}/blobs/xxx 到/{packageName}/blobs/{tag}/xxx + */ +@Component +@EnableConfigurationProperties(OciBlobNodeRefreshJobProperties::class) +class OciBlobNodeRefreshJob( + private val properties: OciBlobNodeRefreshJobProperties, + private val mongoTemplate: MongoTemplate, + private val ociClient: OciClient +) : DefaultContextMongoDbJob(properties) { + private val types: List + get() = properties.repositoryTypes + + override fun start(): Boolean { + return super.start() + } + + override fun entityClass(): Class { + return PackageData::class.java + } + + override fun collectionNames(): List { + return listOf(COLLECTION_NAME) + } + + override fun buildQuery(): Query { + return Query( + Criteria.where(TYPE).`in`(properties.repositoryTypes) + ) + } + + override fun run(row: PackageData, collectionName: String, context: JobContext) { + with(row) { + try { + val result = mongoTemplate.find>( + Query(Criteria(PACKAGE_ID).isEqualTo(row.id)), + PACKAGE_VERSION_NAME + ) + if (result.isEmpty()) return + var refreshStatus = true + result.forEach { + val version = it[NAME] as String? ?: return@with + logger.info( + "Preparing to send blob refresh request for package ${row.name}|${version}" + + " in repo ${row.projectId}|${row.repoName}." + ) + refreshStatus = refreshStatus && ociClient.blobPathRefresh( + projectId = row.projectId, + repoName = row.repoName, + packageName = row.name, + version = version + ).data ?: false + } + if (refreshStatus) { + logger.info( + "Will delete blobs folder of package ${row.name}" + + " in repo ${row.projectId}|${row.repoName}." + ) + ociClient.deleteBlobsFolderAfterRefreshed(projectId, repoName, row.name) + } + } catch (e: Exception) { + throw JobExecuteException( + "Failed to send blob refresh request for package ${row.name}" + + " in repo ${row.projectId}|${row.repoName}, error: ${e.message}", e + ) + } + } + } + + data class PackageData(private val map: Map) { + val id: String by map + val repoName: String by map + val projectId: String by map + val name: String by map + val key: String by map + val type: String by map + } + + companion object { + private val logger = LoggerHolder.jobLogger + const val COLLECTION_NAME = "package" + private const val PACKAGE_VERSION_NAME = "package_version" + private const val METADATA_KEY = "blobPathRefreshed" + private const val METADATA = "metadata" + private const val PACKAGE_ID = "packageId" + private const val NAME = "name" + + } + + override fun mapToEntity(row: Map): PackageData { + return PackageData(row) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt new file mode 100644 index 0000000000..f22e3f3f96 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties("job.oci-blob-node-refresh") +class OciBlobNodeRefreshJobProperties( + /** + * 需要进行远端分发集群推送的仓库类型 + * */ + var repositoryTypes: List = listOf("OCI", "DOCKER"), + override var enabled: Boolean = true, + override var cron: String = "0 0 4/24 * * ?" + ) : MongodbJobProperties() diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt index d3251d105b..b911c0df8a 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt @@ -34,10 +34,12 @@ import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import org.springframework.cloud.openfeign.FeignClient import org.springframework.context.annotation.Primary +import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam @Api("oci") @@ -58,4 +60,21 @@ interface OciClient { @PathVariable projectId: String, @PathVariable repoName: String ): Response + + @ApiOperation("刷新对应版本镜像的blob节点路径") + @PostMapping("/blob/path/refresh/{projectId}/{repoName}") + fun blobPathRefresh( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam packageName: String, + @RequestParam version: String, + ): Response + + @ApiOperation("当历史数据刷新完成后，删除blobs路径下的公共blob节点") + @DeleteMapping("/blobs/delete/{projectId}/{repoName}") + fun deleteBlobsFolderAfterRefreshed( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam packageName: String + ): Response } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index 59f7000686..f48621e608 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -50,6 +50,8 @@ const val N = "n" const val DOCKER_LINK = "Link" const val PROXY_URL = "proxyUrl" +const val BLOB_PATH_VERSION_KEY = "blobPathVersion" +const val BLOB_PATH_REFRESHED_KEY = "blobPathRefreshed" const val MANIFEST = "manifest.json" const val MEDIA_TYPE = "mediaType" diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt index 7cf57840f1..7a1ba8fbcb 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt @@ -78,4 +78,19 @@ class OciPackageController( )) return ResponseBuilder.success() } + + override fun blobPathRefresh(projectId: String, repoName: String, packageName: String, version: String): Response { + return ResponseBuilder.success( + operationService.refreshBlobNode( + projectId = projectId, + repoName = repoName, + pName = packageName, + pVersion = version + )) + } + + override fun deleteBlobsFolderAfterRefreshed(projectId: String, repoName: String, packageName: String): Response { + operationService.deleteBlobsFolderAfterRefreshed(projectId, repoName, packageName) + return ResponseBuilder.success() + } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt index 0f050ed084..3d24080a5f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt @@ -41,7 +41,6 @@ import com.tencent.bkrepo.oci.constant.PAGE_NUMBER import com.tencent.bkrepo.oci.constant.PAGE_SIZE import com.tencent.bkrepo.oci.constant.USER_API_PREFIX import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo -import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.OCI_BLOB_NODE_FULLPATH_REFRESH import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.OCI_PACKAGE_DELETE_URL import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.OCI_USER_MANIFEST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.OCI_USER_REPO_SUFFIX @@ -60,7 +59,6 @@ import io.swagger.annotations.ApiParam import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestAttribute import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam @@ -203,17 +201,4 @@ class UserOciController( ) ) } - - @ApiOperation("刷新oci下的所有blob节点路径") - @PostMapping(OCI_BLOB_NODE_FULLPATH_REFRESH) - @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) - fun refreshFullPathOfBlob( - @RequestParam(required = false) - projectId: String? = null, - @RequestParam(required = false) - repoName: String? = null, - ): Response { - operationService.refreshFullPathOfBlob(projectId, repoName) - return ResponseBuilder.success() - } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index 782db9b951..dd63c325d3 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -165,9 +165,22 @@ interface OciOperationService { /** * oci blob 路径调整，由/packageName/blobs/XXX -> /packageName/blobs/version/XXX */ - fun refreshFullPathOfBlob( - projectId: String? = null, - repoName: String? =null, + fun refreshBlobNode( + projectId: String, + repoName: String, + pName: String, + pVersion: String, + userId: String = SecurityUtils.getUserId() + ): Boolean + + + /** + * 当package下所有版本的blob路径都刷新到新的路径下后，删除/packageName/blobs目录 + */ + fun deleteBlobsFolderAfterRefreshed( + projectId: String, + repoName: String, + pName: String, userId: String = SecurityUtils.getUserId() ) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 2c1c2d287c..b2864193f4 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -35,7 +35,6 @@ import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager -import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.composite.CompositeConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration @@ -53,6 +52,8 @@ import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties +import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY import com.tencent.bkrepo.oci.constant.DESCRIPTION import com.tencent.bkrepo.oci.constant.DOWNLOADS import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_BY @@ -103,10 +104,8 @@ import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest -import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail -import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder import com.tencent.bkrepo.repository.pojo.search.PackageQueryBuilder import com.tencent.devops.plugin.api.PluginManager @@ -390,9 +389,11 @@ class OciOperationServiceImpl( fileInfo: FileInfo?, proxyUrl: String? ): NodeDetail? { - var metadata: List? = null + val metadata: MutableList = mutableListOf( + MetadataModel(key = BLOB_PATH_VERSION_KEY, value = BLOB_PATH_VERSION, system = true) + ) proxyUrl?.let { - metadata = mutableListOf(MetadataModel(key = PROXY_URL, value = proxyUrl, system = true)) + metadata.add(MetadataModel(key = PROXY_URL, value = proxyUrl, system = true)) } val request = buildNodeCreateRequest(ociArtifactInfo, artifactFile, metadata) val nodeDetail = if (fileInfo != null) { @@ -516,6 +517,21 @@ class OciOperationServiceImpl( sourceType = sourceType ) + updateNodeMetaData( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + metadata = metadata + ) + } + + + private fun updateNodeMetaData( + projectId: String, + repoName: String, + fullPath: String, + metadata: Map + ) { val metadataSaveRequest = ObjectBuildUtils.buildMetadataSaveRequest( projectId = projectId, repoName = repoName, @@ -672,8 +688,12 @@ class OciOperationServiceImpl( ) nodeClient.createNode(nodeCreateRequest) } - // 删除临时存储路径节点 /packageName/blobs/xxx - deleteNode(projectId, repoName, fullPath, userId) + val metadataMap = metadataClient.listMetadata(projectId, repoName, fullPath).data + if (metadataMap?.get(BLOB_PATH_VERSION_KEY) != null) { + // 只有当新建的blob路径节点才去删除，历史的由定时任务去刷新然后删除 + // 删除临时存储路径节点 /packageName/blobs/xxx + deleteNode(projectId, repoName, fullPath, userId) + } return true } } @@ -960,87 +980,69 @@ class OciOperationServiceImpl( } } - override fun refreshFullPathOfBlob(projectId: String?, repoName: String?, userId: String) { - if (projectId.isNullOrEmpty()) { - projectClient.listProject().data.orEmpty().forEach { - repositoryClient.listRepo( - projectId = projectId!!, type = RepositoryType.OCI.name - ).data.orEmpty().forEach { - refreshPackage(it, userId) - } - repositoryClient.listRepo( - projectId = projectId, type = RepositoryType.DOCKER.name - ).data.orEmpty().forEach { - refreshPackage(it, userId) - } - } - return - } - if (repoName.isNullOrEmpty()) return + override fun deleteBlobsFolderAfterRefreshed(projectId: String, repoName: String, pName: String, userId: String) { val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") - if (repoInfo.type == RepositoryType.OCI || repoInfo.type == RepositoryType.DOCKER){ - refreshPackage(repoInfo, userId) - } - } - - private fun refreshPackage( - repoInfo: RepositoryInfo, - userId: String, - ) { - with(repoInfo) { - val packageList = packageClient.listAllPackageNames(projectId, name).data ?: return - packageList.forEach { pName -> - packageClient.listAllVersion(projectId, name, pName).data.orEmpty().forEach { pVersion -> - refreshNode( - repoInfo = repoInfo, - pName = pName, - pVersion = pVersion, - userId = userId - ) - } - } - } + val blobsFolderPath = StringPool.SLASH+pName + deleteNode(projectId, repoName, blobsFolderPath, userId) } /** * 调整blob目录 * 从/packageName/blobs/xxx 到/packageName/blobs/version/xxx */ - private fun refreshNode( - repoInfo: RepositoryInfo, + override fun refreshBlobNode( + projectId: String, + repoName: String, pName: String, - pVersion: PackageVersion, + pVersion: String, userId: String - ) { + ): Boolean { + val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") val packageName = PackageKeys.resolveName(repoInfo.type.name.toLowerCase(), pName) - val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion.name) + val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion) logger.info("Manifest $manifestPath will be refreshed") val manifestNode = nodeClient.getNodeDetail( repoInfo.projectId, repoInfo.name, manifestPath - ).data ?: return + ).data ?: return false val storageCredentials = repoInfo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } val manifest = loadManifest( manifestNode.sha256!!, manifestNode.size, storageCredentials ) ?: run { logger.warn("The content of manifest.json ${manifestNode.fullPath} is null, check the mediaType.") - return + return false } val ociArtifactInfo = OciManifestArtifactInfo( projectId = repoInfo.projectId, repoName = repoInfo.name, packageName = packageName, - version = pVersion.name, - reference = pVersion.name, + version = pVersion, + reference = pVersion, isValidDigest = false ) + var refreshStatus = true OciUtils.manifestIterator(manifest).forEach { - doSyncBlob(it, ociArtifactInfo, userId) + refreshStatus = refreshStatus && doSyncBlob(it, ociArtifactInfo, userId) + } + + if (refreshStatus) { + updateNodeMetaData( + projectId = repoInfo.projectId, + repoName = repoInfo.name, + fullPath = manifestPath, + metadata = mapOf(BLOB_PATH_REFRESHED_KEY to true) + ) } - logger.info("Manifest $manifestPath has been successfully refreshed") + logger.info("The status of path $manifestPath refreshed is $refreshStatus") + return refreshStatus } + + + + private fun buildImagePackagePullContext( projectId: String, repoName: String, @@ -1090,5 +1092,6 @@ class OciOperationServiceImpl( companion object { val logger: Logger = LoggerFactory.getLogger(OciOperationServiceImpl::class.java) + private const val BLOB_PATH_VERSION = "v1" } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt index b1132e1d7a..c7a3c534d8 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.oci.constant.DIGEST_LIST import com.tencent.bkrepo.oci.constant.IMAGE_VERSION import com.tencent.bkrepo.oci.constant.MEDIA_TYPE @@ -100,7 +101,8 @@ object ObjectBuildUtils { sourceType: ArtifactChannel? = null ): MutableMap { return mutableMapOf( - MEDIA_TYPE to mediaType + MEDIA_TYPE to mediaType, + BLOB_PATH_REFRESHED_KEY to true ).apply { version?.let { this.put(IMAGE_VERSION, version) } digestList?.let { this.put(DIGEST_LIST, digestList) } From 2508b10af3dfb52eb7e0f07507d404b4a9064c3a Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 23 Aug 2023 11:07:44 +0800 Subject: [PATCH 35/41] =?UTF-8?q?bug:=20=E8=A7=A3=E5=86=B3=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=86=B2=E7=AA=81#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/OciRegistryLocalRepository.kt | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 02ca4ae6bd..320e05b323 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -236,36 +236,36 @@ class OciRegistryLocalRepository( artifactFile = context.getArtifactFile(), storageCredentials = context.repositoryDetail.storageCredentials ) - val fileInfo = storageService.finishAppend(artifactInfo.uuid!!, context.repositoryDetail.storageCredentials) - if (fileInfo.sha256 != sha256) - throw OciBadRequestException(OciMessageCode.OCI_DIGEST_INVALID, sha256) - // 当并发情况下文件被删可能导致文件size为0 - if (fileInfo.size == 0L && fileInfo.sha256 != EMPTY_FILE_SHA256) - throw StorageErrorException(StorageMessageCode.STORE_ERROR) - ociOperationService.storeArtifact( - ociArtifactInfo = context.artifactInfo as OciArtifactInfo, - artifactFile = context.getArtifactFile(), - storageCredentials = context.storageCredentials, - fileInfo = fileInfo - ) - fileInfo - } catch (e: StorageErrorException) { - // 计算sha256和转存文件导致时间较长，会出现请求超时，然后发起重试，导致并发操作该临时文件，文件可能已经被删除 - if (storageService.exist(sha256, context.repositoryDetail.storageCredentials)) { - val nodeDetail = nodeClient.getNodeDetail( - artifactInfo.projectId, artifactInfo.repoName, artifactInfo.getArtifactFullPath() - ).data - if (nodeDetail == null || nodeDetail.sha256 != sha256) { - throw e + val fileInfo = storageService.finishAppend(artifactInfo.uuid!!, context.repositoryDetail.storageCredentials) + if (fileInfo.sha256 != sha256) + throw OciBadRequestException(OciMessageCode.OCI_DIGEST_INVALID, sha256) + // 当并发情况下文件被删可能导致文件size为0 + if (fileInfo.size == 0L && fileInfo.sha256 != EMPTY_FILE_SHA256) + throw StorageErrorException(StorageMessageCode.STORE_ERROR) + ociOperationService.storeArtifact( + ociArtifactInfo = context.artifactInfo as OciArtifactInfo, + artifactFile = context.getArtifactFile(), + storageCredentials = context.storageCredentials, + fileInfo = fileInfo + ) + fileInfo + } catch (e: StorageErrorException) { + // 计算sha256和转存文件导致时间较长，会出现请求超时，然后发起重试，导致并发操作该临时文件，文件可能已经被删除 + if (storageService.exist(sha256, context.repositoryDetail.storageCredentials)) { + val nodeDetail = nodeClient.getNodeDetail( + artifactInfo.projectId, artifactInfo.repoName, artifactInfo.getArtifactFullPath() + ).data + if (nodeDetail == null || nodeDetail.sha256 != sha256) { + throw e + } else { + FileInfo(nodeDetail.sha256!!, nodeDetail.md5!!, nodeDetail.size) + } } else { - FileInfo(nodeDetail.sha256!!, nodeDetail.md5!!, nodeDetail.size) + throw e } - } else { - throw e } - } - val digest = OciDigest.fromSha256(fileInfo.sha256) - val blobLocation = OciLocationUtils.blobLocation(digest, artifactInfo) + val digest = OciDigest.fromSha256(fileInfo.sha256) + val blobLocation = OciLocationUtils.blobLocation(digest, artifactInfo) logger.info( "Artifact ${context.artifactInfo.getArtifactFullPath()} " + "has been uploaded to ${context.artifactInfo.getArtifactFullPath()}" + From 485c13e046966e5f1f5e91e6941844a0bb33d64d Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 23 Aug 2023 11:11:40 +0800 Subject: [PATCH 36/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=A0=81=E6=89=AB?= =?UTF-8?q?=E6=8F=8F=E9=97=AE=E9=A2=98=E4=BF=AE=E5=A4=8D#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/controller/service/OciPackageController.kt | 4 +++- .../bkrepo/oci/service/impl/OciOperationServiceImpl.kt | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt index 7a1ba8fbcb..89b709c23f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt @@ -79,7 +79,9 @@ class OciPackageController( return ResponseBuilder.success() } - override fun blobPathRefresh(projectId: String, repoName: String, packageName: String, version: String): Response { + override fun blobPathRefresh( + projectId: String, repoName: String, packageName: String, version: String + ): Response { return ResponseBuilder.success( operationService.refreshBlobNode( projectId = projectId, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index b2864193f4..64b92a1ce2 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -980,7 +980,9 @@ class OciOperationServiceImpl( } } - override fun deleteBlobsFolderAfterRefreshed(projectId: String, repoName: String, pName: String, userId: String) { + override fun deleteBlobsFolderAfterRefreshed( + projectId: String, repoName: String, pName: String, userId: String + ) { val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") val blobsFolderPath = StringPool.SLASH+pName From c2266a048b9e1caf3164c2870c5ec84fac386dbe Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 24 Aug 2023 17:26:21 +0800 Subject: [PATCH 37/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=A0=81=E9=97=AE?= =?UTF-8?q?=E9=A2=98=E4=BF=AE=E5=A4=8D#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/job/batch/OciBlobNodeRefreshJob.kt | 5 +- .../bkrepo/oci/constant/OciConstants.kt | 1 + .../bkrepo/oci/util/OciLocationUtils.kt | 9 +++- .../service/impl/OciOperationServiceImpl.kt | 52 ++++++++++++------- .../bkrepo/repository/api/NodeClient.kt | 12 ++++- .../controller/service/NodeController.kt | 6 +++ .../service/node/NodeRestoreOperation.kt | 4 ++ .../service/node/impl/NodeRestoreSupport.kt | 7 +++ .../service/node/impl/NodeServiceImpl.kt | 6 +++ .../node/impl/edge/EdgeNodeServiceImpl.kt | 4 ++ .../bkrepo/repository/util/NodeQueryHelper.kt | 11 ++++ 11 files changed, 93 insertions(+), 24 deletions(-) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt index b2a3437e16..68a139804a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt @@ -83,8 +83,8 @@ class OciBlobNodeRefreshJob( ) if (result.isEmpty()) return var refreshStatus = true - result.forEach { - val version = it[NAME] as String? ?: return@with + for (map in result) { + val version = map[NAME] as String? ?: continue logger.info( "Preparing to send blob refresh request for package ${row.name}|${version}" + " in repo ${row.projectId}|${row.repoName}." @@ -97,6 +97,7 @@ class OciBlobNodeRefreshJob( ).data ?: false } if (refreshStatus) { + // 当包下版本对应镜像的 blob节点都刷新完成后，删除旧路径下的 blob文件 logger.info( "Will delete blobs folder of package ${row.name}" + " in repo ${row.projectId}|${row.repoName}." diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index 9e241e152a..323942a45e 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -93,6 +93,7 @@ const val LAST_MODIFIED_BY = "lastModifiedBy" const val LAST_MODIFIED_DATE = "lastModifiedDate" const val DOWNLOADS = "downloads" const val MD5 = "md5" +const val DELETED = "deleted" const val EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index 37de94fa4f..c7b1dbb7c2 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.oci.util +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.oci.constant.OCI_MANIFEST import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest @@ -57,12 +58,16 @@ object OciLocationUtils { return buildPath(packageName, ref, "blobs") } + fun buildBlobsFolderPath(packageName: String): String { + return buildPath(packageName, null, "blobs") + } + fun buildDigestBlobsUploadPath(packageName: String, ref: OciDigest): String { return buildPath(packageName, ref, "_uploads") } - private fun buildPath(packageName: String, ref: OciDigest, type: String): String { - return "/$packageName/$type/" + ref.fileName() + private fun buildPath(packageName: String, ref: OciDigest? = null, type: String): String { + return "/$packageName/$type/"+ (ref?.fileName() ?: StringPool.EMPTY) } fun manifestLocation(digest: OciDigest, ociArtifactInfo: OciArtifactInfo): String { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 64b92a1ce2..fea2d58d78 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -90,13 +90,13 @@ import com.tencent.bkrepo.oci.pojo.user.PackageVersionInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.ObjectBuildUtils import com.tencent.bkrepo.oci.util.OciLocationUtils +import com.tencent.bkrepo.oci.util.OciLocationUtils.buildBlobsFolderPath import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.oci.util.OciUtils import com.tencent.bkrepo.repository.api.MetadataClient import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.api.PackageMetadataClient -import com.tencent.bkrepo.repository.api.ProjectClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER @@ -104,6 +104,7 @@ import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder @@ -134,7 +135,6 @@ class OciOperationServiceImpl( private val storageService: StorageService, private val storageManager: StorageManager, private val repositoryClient: RepositoryClient, - private val projectClient: ProjectClient, private val ociProperties: OciProperties, private val ociReplicationRecordDao: OciReplicationRecordDao, private val storageCredentialsClient: StorageCredentialsClient, @@ -666,12 +666,10 @@ class OciOperationServiceImpl( with(ociArtifactInfo) { // 并发情况下，版本目录下可能存在着非该版本的blob // 覆盖上传时会先删除原有目录，并发情况下可能导致blobs节点不存在 - var nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) - if (nodeProperty == null) { - nodeProperty = nodeClient.getDeletedNodeDetail( - projectId, repoName, fullPath - ).data?.firstOrNull{ it.sha256 == descriptor.sha256 }?.let { - NodeProperty(it.fullPath, it.md5, it.size) + var nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) ?: run { + nodeClient.getDeletedNodeDetailBySha256( + projectId, repoName, descriptor.sha256).data?.let { + NodeProperty(StringPool.EMPTY, it.md5, it.size) } ?: return false } val newPath = OciLocationUtils.blobVersionPathLocation(reference, packageName, descriptor.filename) @@ -985,8 +983,19 @@ class OciOperationServiceImpl( ) { val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") - val blobsFolderPath = StringPool.SLASH+pName - deleteNode(projectId, repoName, blobsFolderPath, userId) + val blobsFolderPath = buildBlobsFolderPath(pName) + val fullPaths = nodeClient.listNode(projectId, repoName, blobsFolderPath, includeFolder = false, deep = false).data?.map { + it.fullPath + } + if (fullPaths.isNullOrEmpty()) return + logger.info("Blobs of package $pName in folder $blobsFolderPath will be deleted in $projectId|$repoName") + val request = NodesDeleteRequest( + projectId = projectId, + repoName = repoName, + fullPaths = fullPaths, + operator = userId + ) + nodeClient.deleteNodes(request) } /** @@ -1005,9 +1014,22 @@ class OciOperationServiceImpl( val packageName = PackageKeys.resolveName(repoInfo.type.name.toLowerCase(), pName) val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion) logger.info("Manifest $manifestPath will be refreshed") + val ociArtifactInfo = OciManifestArtifactInfo( + projectId = repoInfo.projectId, + repoName = repoInfo.name, + packageName = packageName, + version = pVersion, + reference = pVersion, + isValidDigest = false + ) val manifestNode = nodeClient.getNodeDetail( repoInfo.projectId, repoInfo.name, manifestPath - ).data ?: return false + ).data ?: run { + val oldDockerFullPath = getDockerNode(ociArtifactInfo) ?: return false + nodeClient.getNodeDetail( + repoInfo.projectId, repoInfo.name, oldDockerFullPath + ).data ?: return false + } val storageCredentials = repoInfo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } val manifest = loadManifest( manifestNode.sha256!!, manifestNode.size, storageCredentials @@ -1015,14 +1037,6 @@ class OciOperationServiceImpl( logger.warn("The content of manifest.json ${manifestNode.fullPath} is null, check the mediaType.") return false } - val ociArtifactInfo = OciManifestArtifactInfo( - projectId = repoInfo.projectId, - repoName = repoInfo.name, - packageName = packageName, - version = pVersion, - reference = pVersion, - isValidDigest = false - ) var refreshStatus = true OciUtils.manifestIterator(manifest).forEach { refreshStatus = refreshStatus && doSyncBlob(it, ociArtifactInfo, userId) diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt index d8fabdba8d..45afee4fcc 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt @@ -133,7 +133,7 @@ interface NodeClient { @ApiOperation("删除节点") @DeleteMapping("/batch/delete") - fun deleteNodes(nodesDeleteRequest: NodesDeleteRequest): Response + fun deleteNodes(@RequestBody nodesDeleteRequest: NodesDeleteRequest): Response @ApiOperation("恢复节点") @PostMapping("/restore") @@ -194,4 +194,14 @@ interface NodeClient { @PathVariable repoName: String, @RequestParam fullPath: String ): Response> + + @ApiOperation("通过sha256查询已删除节点") + @GetMapping("/deletedBySha256/detail/{projectId}/{repoName}") + fun getDeletedNodeDetailBySha256( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam sha256: String + ): Response + + } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt index f6c3c79e34..c954cf223c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt @@ -178,4 +178,10 @@ class NodeController( val artifactInfo = DefaultArtifactInfo(projectId, repoName, fullPath) return ResponseBuilder.success(nodeService.getDeletedNodeDetail(artifactInfo)) } + + override fun getDeletedNodeDetailBySha256( + projectId: String, repoName: String, sha256: String + ): Response { + return ResponseBuilder.success(nodeService.getDeletedNodeDetailBySha256(projectId, repoName, sha256)) + } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt index 2fa41c70bb..617603e44e 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt @@ -48,6 +48,10 @@ interface NodeRestoreOperation { */ fun getDeletedNodeDetail(artifact: ArtifactInfo): List + /** + * 根据 sha256 查询被删除的节点详情 + */ + fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? /** * 恢复被删除的节点 * @return 恢复节点数量 diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt index 4cfe17bc50..b3eb0d925c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt @@ -50,6 +50,7 @@ import com.tencent.bkrepo.repository.util.MetadataUtils import com.tencent.bkrepo.repository.util.NodeQueryHelper import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedFolderQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointListQuery +import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointListQueryBySha256 import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeListQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeQuery @@ -76,6 +77,12 @@ open class NodeRestoreSupport( } } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + val query = nodeDeletedPointListQueryBySha256(projectId, repoName, sha256) + val node = nodeDao.findOne(query) + return convertToDetail(node) + } + override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { with(resolveContext(artifact, nodeRestoreOption)) { return restoreNode(this) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt index 268d705553..1f3fb9ac84 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt @@ -149,6 +149,12 @@ class NodeServiceImpl( return NodeRestoreSupport(this).getDeletedNodeDetail(artifact) } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + return NodeRestoreSupport(this).getDeletedNodeDetailBySha256( + projectId, repoName, sha256 + ) + } + @Transactional(rollbackFor = [Throwable::class]) override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { return NodeRestoreSupport(this).restoreNode(artifact, nodeRestoreOption) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt index 614a9ea66d..34479b7496 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -162,6 +162,10 @@ class EdgeNodeServiceImpl( return NodeRestoreSupport(this).getDeletedNodeDetail(artifact) } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + return NodeRestoreSupport(this).getDeletedNodeDetailBySha256(projectId, repoName, sha256) + } + @Transactional(rollbackFor = [Throwable::class]) override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { centerNodeClient.restoreNode(NodeRestoreRequest(artifact, nodeRestoreOption)) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt index f370ad00b9..573ff1fbd2 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt @@ -119,6 +119,17 @@ object NodeQueryHelper { return Query(criteria).with(Sort.by(Sort.Direction.DESC, TNode::deleted.name)) } + /** + * 通过sha256查询被删除节点详情 + */ + fun nodeDeletedPointListQueryBySha256(projectId: String, repoName: String, sha256: String): Query { + val criteria = where(TNode::projectId).isEqualTo(projectId) + .and(TNode::repoName).isEqualTo(repoName) + .and(TNode::sha256).isEqualTo(sha256) + .and(TNode::deleted).ne(null) + return Query(criteria).with(Sort.by(Sort.Direction.DESC, TNode::deleted.name)) + } + /** * 查询单个被删除节点 */ From 35fddf8f376824dbbb6b581a9ba5c7ebe671479a Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 24 Aug 2023 22:20:55 +0800 Subject: [PATCH 38/41] =?UTF-8?q?bug:=20=E4=BB=A3=E7=A0=81=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E9=97=AE=E9=A2=98=E4=BF=AE=E5=A4=8D#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oci/controller/service/OciPackageController.kt | 4 +++- .../oci/service/impl/OciOperationServiceImpl.kt | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt index 89b709c23f..8aaaccd54c 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt @@ -91,7 +91,9 @@ class OciPackageController( )) } - override fun deleteBlobsFolderAfterRefreshed(projectId: String, repoName: String, packageName: String): Response { + override fun deleteBlobsFolderAfterRefreshed( + projectId: String, repoName: String, packageName: String + ): Response { operationService.deleteBlobsFolderAfterRefreshed(projectId, repoName, packageName) return ResponseBuilder.success() } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index fea2d58d78..388fc62299 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -620,7 +620,7 @@ class OciOperationServiceImpl( existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, userId) if (!existFlag) { // 第三方同步场景下，如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 - return Pair(existFlag, 0) + return Pair(false, 0) } } return Pair(existFlag, size) @@ -666,7 +666,7 @@ class OciOperationServiceImpl( with(ociArtifactInfo) { // 并发情况下，版本目录下可能存在着非该版本的blob // 覆盖上传时会先删除原有目录，并发情况下可能导致blobs节点不存在 - var nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) ?: run { + val nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) ?: run { nodeClient.getDeletedNodeDetailBySha256( projectId, repoName, descriptor.sha256).data?.let { NodeProperty(StringPool.EMPTY, it.md5, it.size) @@ -981,12 +981,12 @@ class OciOperationServiceImpl( override fun deleteBlobsFolderAfterRefreshed( projectId: String, repoName: String, pName: String, userId: String ) { - val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data + repositoryClient.getRepoInfo(projectId, repoName).data ?: throw RepoNotFoundException("$projectId|$repoName") val blobsFolderPath = buildBlobsFolderPath(pName) - val fullPaths = nodeClient.listNode(projectId, repoName, blobsFolderPath, includeFolder = false, deep = false).data?.map { - it.fullPath - } + val fullPaths = nodeClient.listNode( + projectId, repoName, blobsFolderPath, includeFolder = false, deep = false + ).data?.map { it.fullPath } if (fullPaths.isNullOrEmpty()) return logger.info("Blobs of package $pName in folder $blobsFolderPath will be deleted in $projectId|$repoName") val request = NodesDeleteRequest( From c0a3298b80fc7ea46a812d8da8a8dc5221b30140 Mon Sep 17 00:00:00 2001 From: zacYL Date: Fri, 25 Aug 2023 10:13:50 +0800 Subject: [PATCH 39/41] =?UTF-8?q?bug:=20mount=E4=B8=8A=E4=BC=A0=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E5=85=83=E6=95=B0=E6=8D=AE=EF=BC=8C=E7=94=A8=E4=BA=8E?= =?UTF-8?q?=E5=8C=BA=E5=88=AB=E6=96=B0=E8=80=81=20blob=20=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E8=B7=AF=E5=BE=84#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/oci/constant/OciConstants.kt | 2 ++ .../bkrepo/oci/service/impl/OciBlobServiceImpl.kt | 10 +++++++++- .../bkrepo/oci/service/impl/OciOperationServiceImpl.kt | 5 +++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index 323942a45e..c4e56a21fe 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -51,6 +51,8 @@ const val DOCKER_LINK = "Link" const val PROXY_URL = "proxyUrl" const val BLOB_PATH_VERSION_KEY = "blobPathVersion" +const val BLOB_PATH_VERSION_VALUE = "v1" + const val BLOB_PATH_REFRESHED_KEY = "blobPathRefreshed" const val MANIFEST = "manifest.json" diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index df449104c7..53646481db 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -43,6 +43,8 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadConte import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.REPO_TYPE import com.tencent.bkrepo.oci.exception.OciBadRequestException @@ -56,6 +58,7 @@ import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -139,13 +142,18 @@ class OciBlobServiceImpl( buildSessionIdLocationForUpload(this, domain) return } + // 用于新版本 blobs 路径区分 + val metadata: MutableList = mutableListOf( + MetadataModel(key = BLOB_PATH_VERSION_KEY, value = BLOB_PATH_VERSION_VALUE, system = true) + ) val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( projectId = projectId, repoName = repoName, size = nodeProperty.size!!, sha256 = ociDigest.hex, fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, ociDigest), - md5 = nodeProperty.md5!! + md5 = nodeProperty.md5!!, + metadata = metadata ) nodeClient.createNode(nodeCreateRequest) val blobLocation = OciLocationUtils.blobLocation(ociDigest, this) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 388fc62299..c267c670cb 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -54,6 +54,7 @@ import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE import com.tencent.bkrepo.oci.constant.DESCRIPTION import com.tencent.bkrepo.oci.constant.DOWNLOADS import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_BY @@ -389,8 +390,9 @@ class OciOperationServiceImpl( fileInfo: FileInfo?, proxyUrl: String? ): NodeDetail? { + // 用于新版本 blobs 路径区分， blob存储路径由 /{package}/blobs/转为/{package}/blobs/{version}/ val metadata: MutableList = mutableListOf( - MetadataModel(key = BLOB_PATH_VERSION_KEY, value = BLOB_PATH_VERSION, system = true) + MetadataModel(key = BLOB_PATH_VERSION_KEY, value = BLOB_PATH_VERSION_VALUE, system = true) ) proxyUrl?.let { metadata.add(MetadataModel(key = PROXY_URL, value = proxyUrl, system = true)) @@ -1108,6 +1110,5 @@ class OciOperationServiceImpl( companion object { val logger: Logger = LoggerFactory.getLogger(OciOperationServiceImpl::class.java) - private const val BLOB_PATH_VERSION = "v1" } } From 3bdbcf602931c81de25229ce2ad2935c197b7060 Mon Sep 17 00:00:00 2001 From: zacYL Date: Sun, 27 Aug 2023 20:52:49 +0800 Subject: [PATCH 40/41] =?UTF-8?q?bug:=20=E9=95=9C=E5=83=8F=E5=88=B7?= =?UTF-8?q?=E6=96=B0=E5=89=8D=E5=85=88=E5=88=A4=E6=96=AD=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E5=B7=B2=E7=BB=8F=E6=98=AF=E6=96=B0=E7=89=88=E6=9C=AC=E8=B7=AF?= =?UTF-8?q?=E5=BE=84#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/service/impl/OciOperationServiceImpl.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index c267c670cb..9bb3516df1 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -1032,6 +1032,11 @@ class OciOperationServiceImpl( repoInfo.projectId, repoInfo.name, oldDockerFullPath ).data ?: return false } + val refreshedMetadat = manifestNode.nodeMetadata.firstOrNull { it.key == BLOB_PATH_REFRESHED_KEY} + if (refreshedMetadat != null) { + logger.info("$manifestPath has been refreshed, ignore it") + return true + } val storageCredentials = repoInfo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } val manifest = loadManifest( manifestNode.sha256!!, manifestNode.size, storageCredentials From 171a5bb8cbe08e58463d1cba8cd58872e07b43a5 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Wed, 30 Aug 2023 16:33:18 +0800 Subject: [PATCH 41/41] =?UTF-8?q?feat:=20=E9=95=9C=E5=83=8Fblob=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E8=B0=83=E6=95=B4=E5=90=8E=EF=BC=8C=E5=88=86=E5=8F=91?= =?UTF-8?q?=E5=AF=B9=E5=BA=94=E8=8E=B7=E5=8F=96=E8=B7=AF=E5=BE=84=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/replication/constant/Constants.kt | 2 + .../internal/type/DockerPackageNodeMapper.kt | 40 ++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt index 0c2e8b5be0..cb08bd2168 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt @@ -40,6 +40,8 @@ const val DOCKER_MANIFEST_JSON_FULL_PATH = "/%s/%s/manifest.json" const val DOCKER_LAYER_FULL_PATH = "/%s/%s/%s" const val OCI_MANIFEST_JSON_FULL_PATH = "/%s/manifest/%s/manifest.json" const val OCI_LAYER_FULL_PATH = "/%s/blobs/%s" +const val OCI_LAYER_FULL_PATH_V1 = "/%s/blobs/%s/%s" +const val BLOB_PATH_REFRESHED_KEY = "blobPathRefreshed" const val NODE_FULL_PATH = "fullPath" const val SIZE = "size" const val REPOSITORY_INFO = "repo" diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt index 5df18e0027..bfb0450916 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt @@ -4,13 +4,16 @@ import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.replication.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.replication.constant.DOCKER_LAYER_FULL_PATH import com.tencent.bkrepo.replication.constant.DOCKER_MANIFEST_JSON_FULL_PATH import com.tencent.bkrepo.replication.constant.OCI_LAYER_FULL_PATH +import com.tencent.bkrepo.replication.constant.OCI_LAYER_FULL_PATH_V1 import com.tencent.bkrepo.replication.constant.OCI_MANIFEST_JSON_FULL_PATH import com.tencent.bkrepo.replication.util.ManifestParser import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.springframework.stereotype.Component @@ -61,15 +64,40 @@ class DockerPackageNodeMapper( throw ArtifactNotFoundException("Could not read manifest.json, $e") } manifestInfo!!.descriptors?.forEach { - val replace = it.replace(":", "__") - if (isOci) { - result.add(OCI_LAYER_FULL_PATH.format(name, replace)) - } else { - result.add(DOCKER_LAYER_FULL_PATH.format(name, version, replace)) - } + result.add(buildBlobPath( + descriptor = it, + packageName = name, + version = version, + isOci = isOci, + nodeDetail = nodeDetail + )) } result.add(manifestFullPath) return result } } + + /** + * 通过包名、版本、sha256拼接出blob路径 + */ + private fun buildBlobPath( + descriptor: String, + packageName: String, + version: String, + isOci: Boolean, + nodeDetail: NodeDetail + ): String { + val replace = descriptor.replace(":", "__") + return if (isOci) { + // 镜像blob路径格式有调整，从/package/blobs/下调至//package/blobs/version/ + val refreshedMetadata = nodeDetail.nodeMetadata.firstOrNull { it.key == BLOB_PATH_REFRESHED_KEY} + if (refreshedMetadata != null) { + OCI_LAYER_FULL_PATH_V1.format(packageName, version, replace) + } else { + OCI_LAYER_FULL_PATH.format(packageName, replace) + } + } else { + DOCKER_LAYER_FULL_PATH.format(packageName, version, replace) + } + } }