feat: 镜像制品删除优化 #203

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt

@@ -0,0 +1,34 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.api.constant
+
+object AuthenticationKeys {
+    const val BEARER_REALM = "Bearer realm"
+    const val SERVICE = "service"
+    const val SCOPE = "scope"
+}

src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt → src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.replication.pojo.remote
+package com.tencent.bkrepo.common.api.pojo.authentication
 
 /**
  * 返回头中的WWW_AUTHENTICATE字段包含的属性

src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt

@@ -0,0 +1,74 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.api.util
+
+import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.BEARER_REALM
+import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SCOPE
+import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SERVICE
+import com.tencent.bkrepo.common.api.constant.StringPool
+import com.tencent.bkrepo.common.api.pojo.authentication.AuthenticationProperty
+
+object AuthenticationUtil {
+
+    /**
+     * 解析返回头中的WWW_AUTHENTICATE字段， 只针对为Bearer realm
+     */
+    fun parseWWWAuthenticateHeader(wwwAuthenticate: String, scope: String?): AuthenticationProperty? {
+        val map: MutableMap<String, String> = mutableMapOf()
+        return try {
+            val params = wwwAuthenticate.split("\",")
+            params.forEach {
+                val param = it.split(Regex("="),2)
+                val name = param.first()
+                val value = param.last().replace("\"", "")
+                map[name] = value
+            }
+            AuthenticationProperty(
+                authUrl = map[BEARER_REALM]!!,
+                service = map[SERVICE]!!,
+                scope = scope
+            )
+        } catch (e: Exception) {
+            null
+        }
+    }
+
+    fun buildAuthenticationUrl(property: AuthenticationProperty, userName: String?): String? {
+        if (property.authUrl.isBlank()) return null
+        var result = if (property.authUrl.contains(StringPool.QUESTION)) {
+            "${property.authUrl}&$SERVICE=${property.service}"
+        } else {
+            "${property.authUrl}?$SERVICE=${property.service}"
+        }
+        property.scope?.let {
+            result += "&$SCOPE=${property.scope}"
+        }
+        userName?.let { result += "&account=$userName" }
+        return result
+    }
+}

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt

@@ -233,6 +233,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
         if (addInterceptor) {
             createAuthenticateInterceptor(configuration.credentials)?.let { builder.addInterceptor(it) }
         }
+        builder.retryOnConnectionFailure(true)
         return builder.build()
     }
 

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt

@@ -31,10 +31,16 @@
 
 package com.tencent.bkrepo.common.artifact.util.http
 
+import com.tencent.bkrepo.common.api.constant.CharPool
 import com.tencent.bkrepo.common.api.constant.CharPool.QUESTION
 import com.tencent.bkrepo.common.api.constant.CharPool.SLASH
+import com.tencent.bkrepo.common.api.constant.StringPool
 import com.tencent.bkrepo.common.api.constant.StringPool.HTTP
 import com.tencent.bkrepo.common.api.constant.StringPool.HTTPS
+import com.tencent.bkrepo.common.storage.innercos.retry
+import java.net.HttpURLConnection
+import java.net.MalformedURLException
+import java.net.URL
 
 /**
  * Http URL 格式化工具类
@@ -82,4 +88,87 @@ object UrlFormatter {
         }
         return url
     }
+
+
+    /**
+     * 拼接url
+     */
+    fun buildUrl(
+        url: String,
+        path: String = StringPool.EMPTY,
+        params: String = StringPool.EMPTY,
+    ): String {
+        if (url.isBlank())
+            throw IllegalArgumentException("Url should not be blank")
+        val newUrl = addProtocol(url.trim().trimEnd(SLASH))
+        val baseUrl = URL(newUrl, newUrl.path)
+        val builder = StringBuilder(baseUrl.toString().trimEnd(SLASH))
+        if (path.isNotBlank()) {
+            builder.append(SLASH).append(path.trimStart(SLASH))
+        }
+        if (!newUrl.query.isNullOrEmpty()) {
+            builder.append(QUESTION).append(newUrl.query)
+        }
+        return addParams(builder.toString(), params)
+    }
+
+    fun addParams(url: String, params: String): String {
+        val baseUrl = URL(url)
+        val builder = StringBuilder(baseUrl.toString())
+
+        if (params.isNotEmpty()) {
+            if (builder.contains(QUESTION)) {
+                builder.append(CharPool.AND).append(params)
+            } else {
+                builder.append(QUESTION).append(params)
+            }
+        }
+        return builder.toString()
+    }
+
+    /**
+     * 当没有protocol时进行添加
+     */
+    fun addProtocol(registry: String): URL {
+        try {
+            return URL(registry)
+        } catch (ignore: MalformedURLException) {
+        }
+        return addProtocolToHost(registry)
+    }
+
+    /**
+     * 针对url如果没传protocol， 则默认以https请求发送；
+     * 如果http请求无法访问，则以http发送
+     */
+    private fun addProtocolToHost(registry: String): URL {
+        val url = try {
+            URL("$HTTPS$registry")
+        } catch (ignore: MalformedURLException) {
+            throw IllegalArgumentException("Check your input url!")
+        }
+        return try {
+            retry(times = 3, delayInSeconds = 1) {
+                validateHttpsProtocol(url)
+                url
+            }
+        } catch (ignore: Exception) {
+            URL(url.toString().replaceFirst("^https".toRegex(), "http"))
+        }
+    }
+
+    /**
+     * 验证registry是否支持https
+     */
+    private fun validateHttpsProtocol(url: URL): Boolean {
+        return try {
+            val http: HttpURLConnection = url.openConnection() as HttpURLConnection
+            http.instanceFollowRedirects = false
+            http.responseCode
+            http.disconnect()
+            true
+        } catch (e: Exception) {
+            throw e
+        }
+    }
 }