Merge remote-tracking branch 'github/master' into issue_1064

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt

@@ -0,0 +1,34 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.api.constant
+
+object AuthenticationKeys {
+    const val BEARER_REALM = "Bearer realm"
+    const val SERVICE = "service"
+    const val SCOPE = "scope"
+}

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/exception/TooManyRequestsException.kt

@@ -0,0 +1,38 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.api.exception
+
+import com.tencent.bkrepo.common.api.constant.HttpStatus
+import com.tencent.bkrepo.common.api.message.CommonMessageCode
+
+/**
+ * 请求过多，超出了频次限制
+ */
+class TooManyRequestsException(
+    parameter: String = "Too Many Requests"
+) : ErrorCodeException(HttpStatus.TOO_MANY_REQUESTS, CommonMessageCode.TOO_MANY_REQUESTS, arrayOf(parameter))

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt

@@ -55,6 +55,7 @@ enum class CommonMessageCode(private val key: String) : MessageCode {
     MODIFY_PASSWORD_FAILED("modify.password.failed"),
     OPERATION_CROSS_CLUSTER_NOT_ALLOWED("operation.cross-cluster.not-allowed"),
     MEDIA_TYPE_UNACCEPTABLE("system.media-type.unacceptable"),
+    TOO_MANY_REQUESTS("too.many.requests")
     ;
 
     override fun getBusinessCode() = ordinal + 1

src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt → src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.replication.pojo.remote
+package com.tencent.bkrepo.common.api.pojo.authentication
 
 /**
  * 返回头中的WWW_AUTHENTICATE字段包含的属性

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt

@@ -0,0 +1,74 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.api.util
+
+import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.BEARER_REALM
+import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SCOPE
+import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SERVICE
+import com.tencent.bkrepo.common.api.constant.StringPool
+import com.tencent.bkrepo.common.api.pojo.authentication.AuthenticationProperty
+
+object AuthenticationUtil {
+
+    /**
+     * 解析返回头中的WWW_AUTHENTICATE字段， 只针对为Bearer realm
+     */
+    fun parseWWWAuthenticateHeader(wwwAuthenticate: String, scope: String?): AuthenticationProperty? {
+        val map: MutableMap<String, String> = mutableMapOf()
+        return try {
+            val params = wwwAuthenticate.split("\",")
+            params.forEach {
+                val param = it.split(Regex("="),2)
+                val name = param.first()
+                val value = param.last().replace("\"", "")
+                map[name] = value
+            }
+            AuthenticationProperty(
+                authUrl = map[BEARER_REALM]!!,
+                service = map[SERVICE]!!,
+                scope = scope
+            )
+        } catch (e: Exception) {
+            null
+        }
+    }
+
+    fun buildAuthenticationUrl(property: AuthenticationProperty, userName: String?): String? {
+        if (property.authUrl.isBlank()) return null
+        var result = if (property.authUrl.contains(StringPool.QUESTION)) {
+            "${property.authUrl}&$SERVICE=${property.service}"
+        } else {
+            "${property.authUrl}?$SERVICE=${property.service}"
+        }
+        property.scope?.let {
+            result += "&$SCOPE=${property.scope}"
+        }
+        userName?.let { result += "&account=$userName" }
+        return result
+    }
+}

src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt

@@ -51,6 +51,11 @@ const val ARTIFACT_CONFIGURER = "artifact-configurer"
  */
 const val NODE_DETAIL_KEY = "node-detail"
 
+/**
+ * 查询仓库上传下载限速配置后后将其写入request attributes的key
+ */
+const val REPO_RATE_LIMIT_KEY = "repo-rate-limit"
+
 /**
  * 项目id字段
  */

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetrics.kt

@@ -59,9 +59,11 @@ class ArtifactMetrics(
     init {
         Companion.tagProvider = tagProvider
         Companion.meterRegistry = meterRegistry
+        Companion.properties = properties
         lruMeterFilter = LruMeterFilter(METER_LIMIT_PREFIX, meterRegistry, properties.maxMeters)
         meterRegistry.config().meterFilter(lruMeterFilter)
     }
+
     override fun bindTo(meterRegistry: MeterRegistry) {
         Gauge.builder(ARTIFACT_UPLOADING_COUNT, uploadingCount) { it.get().toDouble() }
             .description(ARTIFACT_UPLOADING_COUNT_DESC)
@@ -94,6 +96,7 @@ class ArtifactMetrics(
         private lateinit var tagProvider: ArtifactTransferTagProvider
         private lateinit var meterRegistry: MeterRegistry
         private lateinit var lruMeterFilter: LruMeterFilter
+        private lateinit var properties: ArtifactMetricsProperties
         private const val MAX_ATIME = 30.0
         private const val BYTES = "bytes"
         private const val DAY = "day"
@@ -107,6 +110,7 @@ class ArtifactMetrics(
                 .description(ARTIFACT_UPLOADED_SIZE_DESC)
                 .baseUnit(BYTES)
                 .publishPercentileHistogram()
+                .maximumExpectedValue(properties.maxLe)
                 .register(meterRegistry)
         }
 
@@ -119,6 +123,7 @@ class ArtifactMetrics(
                 .description(ARTIFACT_DOWNLOADED_SIZE_DESC)
                 .baseUnit(BYTES)
                 .publishPercentileHistogram()
+                .maximumExpectedValue(properties.maxLe)
                 .register(meterRegistry)
         }
 

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactMetricsProperties.kt

@@ -28,6 +28,7 @@
 package com.tencent.bkrepo.common.artifact.metrics
 
 import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.util.unit.DataSize
 
 @ConfigurationProperties("management.metrics")
 data class ArtifactMetricsProperties(
@@ -43,5 +44,9 @@ data class ArtifactMetricsProperties(
     /**
      * 是否通过日志清洗获取传输指标数据
      */
-    var collectByLog: Boolean = false
+    var collectByLog: Boolean = false,
+    /**
+     * 直方图le的最大值
+     */
+    var maxLe: Double = DataSize.ofMegabytes(100).toBytes().toDouble()
 )

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/context/ArtifactContextHolder.kt

@@ -41,18 +41,21 @@ import com.tencent.bkrepo.common.artifact.constant.NODE_DETAIL_KEY
 import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID
 import com.tencent.bkrepo.common.artifact.constant.REPO_KEY
 import com.tencent.bkrepo.common.artifact.constant.REPO_NAME
+import com.tencent.bkrepo.common.artifact.constant.REPO_RATE_LIMIT_KEY
 import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException
 import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory
 import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
 import com.tencent.bkrepo.common.artifact.repository.composite.CompositeRepository
 import com.tencent.bkrepo.common.artifact.repository.core.ArtifactRepository
 import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurity
 import com.tencent.bkrepo.common.service.util.HttpContextHolder
+import com.tencent.bkrepo.common.storage.core.config.RateLimitProperties
 import com.tencent.bkrepo.repository.api.NodeClient
 import com.tencent.bkrepo.repository.api.RepositoryClient
 import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail
 import org.springframework.beans.factory.ObjectProvider
+import org.springframework.util.unit.DataSize
 import org.springframework.web.servlet.HandlerMapping
 import java.util.concurrent.TimeUnit
 import javax.servlet.http.HttpServletRequest
@@ -85,6 +88,10 @@ class ArtifactContextHolder(
         private lateinit var nodeClient: NodeClient
         private lateinit var httpAuthSecurity: ObjectProvider<HttpAuthSecurity>
 
+
+        private const val RECEIVE_RATE_LIMIT_OF_REPO = "receiveRateLimit"
+        private const val RESPONSE_RATE_LIMIT_OF_REPO = "responseRateLimit"
+
         private val artifactConfigurerMap = mutableMapOf<RepositoryType, ArtifactConfigurer>()
         private val repositoryDetailCache = CacheBuilder.newBuilder()
             .maximumSize(1000)
@@ -277,6 +284,36 @@ class ArtifactContextHolder(
             nodeDetail?.let { request.setAttribute(attrKey, nodeDetail) }
             return nodeDetail
         }
+
+        /**
+         * 获取仓库级别的限速配置
+         */
+        fun getRateLimitOfRepo(): RateLimitProperties {
+            val request = HttpContextHolder.getRequestOrNull() ?: return RateLimitProperties()
+            val repoRateLimitAttribute = request.getAttribute(REPO_RATE_LIMIT_KEY)
+            if (repoRateLimitAttribute != null) {
+                require(repoRateLimitAttribute is RateLimitProperties)
+                return repoRateLimitAttribute
+            }
+            // 临时下载链接可能会使用 generic 下载其他业务类型的制品，此处先避免仓库找不到异常
+            // 此处修改潜在风险：当其他类型仓库使用 generic 服务的临时下载链接下载时无法控制住
+            val repo = getRepoDetailOrNull() ?: return RateLimitProperties()
+            val receiveRateLimit = convertToDataSize(repo.configuration.getStringSetting(RECEIVE_RATE_LIMIT_OF_REPO))
+            val responseRateLimit = convertToDataSize(repo.configuration.getStringSetting(RESPONSE_RATE_LIMIT_OF_REPO))
+            val rateLimitProperties = RateLimitProperties(receiveRateLimit, responseRateLimit)
+            request.setAttribute(REPO_RATE_LIMIT_KEY, rateLimitProperties)
+            return rateLimitProperties
+
+        }
+
+        private fun convertToDataSize(dataSize: String?): DataSize {
+            if (dataSize.isNullOrEmpty()) return DataSize.ofBytes(-1)
+            return try {
+                DataSize.parse(dataSize)
+            } catch (e: Exception) {
+                DataSize.ofBytes(-1)
+            }
+        }
     }
 
     /**

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt

@@ -233,6 +233,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
         if (addInterceptor) {
             createAuthenticateInterceptor(configuration.credentials)?.let { builder.addInterceptor(it) }
         }
+        builder.retryOnConnectionFailure(true)
         return builder.build()
     }
 

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt

@@ -31,11 +31,13 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders
 import com.tencent.bkrepo.common.api.constant.HttpStatus
 import com.tencent.bkrepo.common.api.constant.MediaTypes
 import com.tencent.bkrepo.common.api.constant.StringPool
+import com.tencent.bkrepo.common.api.exception.TooManyRequestsException
 import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_MD5
 import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_SHA256
 import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException
 import com.tencent.bkrepo.common.artifact.metrics.RecordAbleInputStream
 import com.tencent.bkrepo.common.artifact.path.PathUtils
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
 import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
 import com.tencent.bkrepo.common.artifact.stream.Range
 import com.tencent.bkrepo.common.artifact.stream.STREAM_BUFFER_SIZE
@@ -50,6 +52,7 @@ import com.tencent.bkrepo.common.storage.monitor.Throughput
 import com.tencent.bkrepo.common.storage.monitor.measureThroughput
 import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import org.springframework.http.HttpMethod
+import org.springframework.util.unit.DataSize
 import java.io.IOException
 import java.time.LocalDateTime
 import java.time.ZoneOffset
@@ -68,6 +71,7 @@ open class DefaultArtifactResourceWriter(
 
     @Throws(ArtifactResponseException::class)
     override fun write(resource: ArtifactResource): Throughput {
+        responseRateLimitCheck()
         return if (resource.containsMultiArtifact()) {
             writeMultiArtifact(resource)
         } else {
@@ -185,7 +189,7 @@ open class DefaultArtifactResourceWriter(
         val recordAbleInputStream = RecordAbleInputStream(inputStream)
         try {
             return measureThroughput {
-                recordAbleInputStream.rateLimit(storageProperties.response.rateLimit.toBytes()).use {
+                recordAbleInputStream.rateLimit(responseRateLimitWrapper(storageProperties.response.rateLimit)).use {
                     it.copyTo(
                         out = response.outputStream,
                         bufferSize = getBufferSize(inputStream.range.length.toInt())
@@ -222,7 +226,9 @@ open class DefaultArtifactResourceWriter(
                     resource.artifactMap.forEach { (name, inputStream) ->
                         val recordAbleInputStream = RecordAbleInputStream(inputStream)
                         zipOutput.putNextEntry(generateZipEntry(name, inputStream))
-                        recordAbleInputStream.rateLimit(storageProperties.response.rateLimit.toBytes()).use {
+                        recordAbleInputStream.rateLimit(
+                            responseRateLimitWrapper(storageProperties.response.rateLimit)
+                        ).use {
                             it.copyTo(
                                 out = zipOutput,
                                 bufferSize = getBufferSize(inputStream.range.length.toInt())
@@ -242,6 +248,31 @@ open class DefaultArtifactResourceWriter(
         }
     }
 
+    /**
+     * 将仓库级别的限速配置导入
+     * 当同时存在全局限速配置以及仓库级别限速配置时，以仓库级别配置优先
+     */
+    private fun responseRateLimitWrapper(rateLimit: DataSize): Long {
+        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
+        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1)) {
+            return rateLimitOfRepo.responseRateLimit.toBytes()
+        }
+        return rateLimit.toBytes()
+    }
+
+    /**
+     * 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接
+     */
+    private fun responseRateLimitCheck() {
+        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
+        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1) &&
+            rateLimitOfRepo.responseRateLimit <= storageProperties.response.circuitBreakerThreshold) {
+            throw TooManyRequestsException(
+                "The circuit breaker is activated when too many download requests are made to the service!"
+            )
+        }
+    }
+
     /**
      * 判断charset,一些媒体类型设置了charset会影响其表现，如application/vnd.android.package-archive
      * */