From df06f906f71e82dbacdcd8e6438c842ba7bd30c0 Mon Sep 17 00:00:00 2001
From: yaoxuwan
Date: Fri, 10 Nov 2023 15:18:19 +0800
Subject: [PATCH 02/74] =?UTF-8?q?feat:=20fs-server=E4=BA=91=E7=A0=94?=
=?UTF-8?q?=E5=8F=91=E5=BF=AB=E6=8D=B7=E7=99=BB=E5=BD=95=20#1389=20(#1415)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: fs-server云研发快捷登录 #1389
* feat: fs-server云研发快捷登录 #1389
---
.../interceptor/devx/DevXWorkSpace.kt | 2 +
.../fs/server/pojo/DevxLoginResponse.kt | 33 ++++
.../fs/server/config/BeanConfiguration.kt | 2 +
.../fs/server/config/RouteConfiguration.kt | 1 +
.../filter/AuthHandlerFilterFunction.kt | 2 +-
.../fs/server/filter/DevXAccessFilter.kt | 76 +--------
.../server/filter/PermissionFilterFunction.kt | 6 +-
.../bkrepo/fs/server/handler/LoginHandler.kt | 11 ++
.../fs/server/utils/DevxWorkspaceUtils.kt | 151 ++++++++++++++++++
.../repo/impl/RepositoryServiceImpl.kt | 2 +-
10 files changed, 208 insertions(+), 78 deletions(-)
create mode 100644 src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/pojo/DevxLoginResponse.kt
create mode 100644 src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt
diff --git a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXWorkSpace.kt b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXWorkSpace.kt
index 3e57587ca0..bd7595f4ac 100644
--- a/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXWorkSpace.kt
+++ b/src/backend/common/common-security/src/main/kotlin/com/tencent/bkrepo/common/security/interceptor/devx/DevXWorkSpace.kt
@@ -36,6 +36,8 @@ data class DevXWorkSpace(
val projectId: String,
@JsonProperty("creator")
val creator: String,
+ @JsonProperty("owner")
+ val owner: String,
@JsonProperty("region_id")
val regionId: String,
@JsonProperty("inner_ip")
diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/pojo/DevxLoginResponse.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/pojo/DevxLoginResponse.kt
new file mode 100644
index 0000000000..b15c000d0e
--- /dev/null
+++ b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/pojo/DevxLoginResponse.kt
@@ -0,0 +1,33 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.fs.server.pojo
+
+data class DevxLoginResponse(
+ val projectId: String,
+ val token: String
+)
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/BeanConfiguration.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/BeanConfiguration.kt
index 4711d71390..98830d7411 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/BeanConfiguration.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/BeanConfiguration.kt
@@ -50,6 +50,7 @@ import com.tencent.bkrepo.fs.server.service.FileOperationService
import com.tencent.bkrepo.fs.server.service.PermissionService
import com.tencent.bkrepo.fs.server.storage.CoArtifactFileFactory
import com.tencent.bkrepo.fs.server.storage.CoStorageManager
+import com.tencent.bkrepo.fs.server.utils.DevxWorkspaceUtils
import com.tencent.bkrepo.fs.server.utils.SecurityManager
import com.tencent.bkrepo.fs.server.utils.SpringContextUtils
import com.tencent.devops.service.config.ServiceProperties
@@ -89,6 +90,7 @@ val beans = beans {
bean()
bean()
bean()
+ bean()
bean {
RouteConfiguration(ref(), ref(), ref(), ref(), ref(), ref(), ref(), ref(), ref(), ref()).router()
}
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt
index c99478307f..3965223ca6 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt
@@ -79,6 +79,7 @@ class RouteConfiguration(
before(RouteConfiguration::initArtifactContext)
filter(permissionFilterFunction::filter)
POST("/login/{projectId}/{repoName}", loginHandler::login)
+ POST("/devx/login/{repoName}", loginHandler::devxLogin)
POST("/token/refresh/{projectId}/{repoName}", loginHandler::refresh)
"/service/block".nest {
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/AuthHandlerFilterFunction.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/AuthHandlerFilterFunction.kt
index 2d1d7cc029..a7f72cc1a7 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/AuthHandlerFilterFunction.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/AuthHandlerFilterFunction.kt
@@ -48,7 +48,7 @@ class AuthHandlerFilterFunction(
request: ServerRequest,
next: suspend (ServerRequest) -> ServerResponse
): ServerResponse {
- if (request.path().startsWith("/login")) {
+ if (request.path().startsWith("/login") || request.path().startsWith("/devx/login")) {
return next(request)
}
var user = ANONYMOUS_USER
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/DevXAccessFilter.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/DevXAccessFilter.kt
index 894364da28..5cbdcaa761 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/DevXAccessFilter.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/DevXAccessFilter.kt
@@ -27,59 +27,28 @@
package com.tencent.bkrepo.fs.server.filter
-import com.google.common.cache.CacheBuilder
-import com.google.common.cache.CacheLoader
-import com.google.common.cache.LoadingCache
import com.tencent.bkrepo.common.api.exception.SystemErrorException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.api.util.IpUtils
-import com.tencent.bkrepo.common.api.util.toJsonString
import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID
import com.tencent.bkrepo.common.security.exception.PermissionException
-import com.tencent.bkrepo.common.security.interceptor.devx.ApiAuth
import com.tencent.bkrepo.common.security.interceptor.devx.DevXProperties
-import com.tencent.bkrepo.common.security.interceptor.devx.QueryResponse
import com.tencent.bkrepo.fs.server.context.ReactiveRequestContextHolder
+import com.tencent.bkrepo.fs.server.utils.DevxWorkspaceUtils
import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils
import kotlinx.coroutines.reactor.awaitSingle
-import kotlinx.coroutines.reactor.mono
import org.slf4j.LoggerFactory
-import org.springframework.http.HttpStatus
-import org.springframework.http.client.reactive.ReactorClientHttpConnector
-import org.springframework.web.reactive.function.client.ClientResponse
-import org.springframework.web.reactive.function.client.WebClient
-import org.springframework.web.reactive.function.client.awaitBody
import org.springframework.web.reactive.function.server.ServerRequest
import org.springframework.web.reactive.function.server.ServerResponse
-import reactor.core.publisher.Mono
-import reactor.netty.http.client.HttpClient
-import reactor.netty.http.client.PrematureCloseException
-import reactor.netty.resources.ConnectionProvider
-import reactor.util.retry.RetryBackoffSpec
-import java.time.Duration
class DevXAccessFilter(
private val devXProperties: DevXProperties
) : CoHandlerFilterFunction {
- private val httpClient by lazy {
- val provider = ConnectionProvider.builder("DevX").maxIdleTime(Duration.ofSeconds(30L)).build()
- val client = HttpClient.create(provider).responseTimeout(Duration.ofSeconds(30L))
- val connector = ReactorClientHttpConnector(client)
- WebClient.builder().clientConnector(connector).build()
- }
- private val projectIpsCache: LoadingCache>> = CacheBuilder.newBuilder()
- .maximumSize(devXProperties.cacheSize)
- .expireAfterWrite(devXProperties.cacheExpireTime)
- .build(CacheLoader.from { key -> listIpFromProject(key) })
-
override suspend fun filter(
request: ServerRequest,
next: suspend (ServerRequest) -> ServerResponse
): ServerResponse {
- if (request.path().startsWith("/login") ||
- request.path().startsWith("/service") ||
- request.path().startsWith("/token")
- ) {
+ if (uncheckedUrlPrefixList.any { request.path().startsWith(it) }) {
return next(request)
}
@@ -122,7 +91,7 @@ class DevXAccessFilter(
}
private suspend fun checkIpBelongToProject(projectId: String, srcIp: String) {
- val projectIps = projectIpsCache.get(projectId).awaitSingle()
+ val projectIps = DevxWorkspaceUtils.getIpList(projectId).awaitSingle()
if (srcIp !in projectIps && !projectIps.any { it.contains('/') && IpUtils.isInRange(srcIp, it) }) {
logger.info("Illegal src ip[$srcIp] in project[$projectId].")
throw PermissionException()
@@ -139,46 +108,9 @@ class DevXAccessFilter(
}
}
- private fun listIpFromProject(projectId: String): Mono> {
- val apiAuth = ApiAuth(devXProperties.appCode, devXProperties.appSecret)
- val token = apiAuth.toJsonString().replace(System.lineSeparator(), "")
- val workspaceUrl = devXProperties.workspaceUrl
-
- logger.info("Update project[$projectId] ips.")
- return httpClient
- .get()
- .uri("$workspaceUrl?project_id=$projectId")
- .header("X-Bkapi-Authorization", token)
- .exchangeToMono {
- mono { parseResponse(it, projectId) }
- }
- .retryWhen(
- RetryBackoffSpec
- .backoff(2L, Duration.ofSeconds(1))
- .filter {
- val retry = it.cause is PrematureCloseException
- logger.warn("request ips of project[$projectId] failed, will retry: $retry")
- retry
- }
- )
- }
-
- private suspend fun parseResponse(response: ClientResponse, projectId: String): Set {
- return if (response.statusCode() != HttpStatus.OK) {
- val errorMsg = response.awaitBody()
- logger.error("${response.statusCode()} $errorMsg")
- devXProperties.projectCvmWhiteList[projectId] ?: emptySet()
- } else {
- val ips = HashSet()
- devXProperties.projectCvmWhiteList[projectId]?.let { ips.addAll(it) }
- response.awaitBody().data.forEach { workspace ->
- workspace.innerIp?.substringAfter('.')?.let { ips.add(it) }
- }
- ips
- }
- }
companion object {
private val logger = LoggerFactory.getLogger(DevXAccessFilter::class.java)
+ private val uncheckedUrlPrefixList = listOf("/login", "/devx/login", "/service", "/token")
}
}
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/PermissionFilterFunction.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/PermissionFilterFunction.kt
index 5552359ce1..c29cd62616 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/PermissionFilterFunction.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/filter/PermissionFilterFunction.kt
@@ -46,10 +46,7 @@ class PermissionFilterFunction(private val securityManager: SecurityManager) : C
request: ServerRequest,
next: suspend (ServerRequest) -> ServerResponse,
): ServerResponse {
- if (request.path().startsWith("/login") ||
- request.path().startsWith("/service") ||
- request.path().startsWith("/token")
- ) {
+ if (uncheckedUrlPrefixList.any { request.path().startsWith(it) }) {
return next(request)
}
val action = request.getAction()
@@ -97,5 +94,6 @@ class PermissionFilterFunction(private val securityManager: SecurityManager) : C
"/node/set-length/**",
"/block/**",
)
+ private val uncheckedUrlPrefixList = listOf("/login", "/devx/login", "/service", "/token")
}
}
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt
index 2dd3133128..2bcce18bc0 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt
@@ -38,10 +38,13 @@ import com.tencent.bkrepo.fs.server.api.RAuthClient
import com.tencent.bkrepo.fs.server.constant.JWT_CLAIMS_PERMIT
import com.tencent.bkrepo.fs.server.constant.JWT_CLAIMS_REPOSITORY
import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder
+import com.tencent.bkrepo.fs.server.pojo.DevxLoginResponse
import com.tencent.bkrepo.fs.server.service.PermissionService
+import com.tencent.bkrepo.fs.server.utils.DevxWorkspaceUtils
import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder
import com.tencent.bkrepo.fs.server.utils.SecurityManager
import kotlinx.coroutines.reactor.awaitSingle
+import kotlinx.coroutines.reactor.awaitSingleOrNull
import org.springframework.web.reactive.function.server.ServerRequest
import org.springframework.web.reactive.function.server.ServerResponse
@@ -75,6 +78,14 @@ class LoginHandler(
return ReactiveResponseBuilder.success(token)
}
+ suspend fun devxLogin(request: ServerRequest): ServerResponse {
+ val workspace = DevxWorkspaceUtils.getWorkspace().awaitSingleOrNull() ?: throw AuthenticationException()
+ val repoName = request.pathVariable(REPO_NAME)
+ val token = createToken(workspace.projectId, repoName, workspace.owner)
+ val response = DevxLoginResponse(workspace.projectId, token)
+ return ReactiveResponseBuilder.success(response)
+ }
+
private suspend fun createToken(projectId: String, repoName: String, username: String): String {
val claims = mutableMapOf(JWT_CLAIMS_REPOSITORY to "$projectId/$repoName")
val writePermit = permissionService.checkPermission(projectId, repoName, PermissionAction.WRITE, username)
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt
new file mode 100644
index 0000000000..8523a3ba0a
--- /dev/null
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt
@@ -0,0 +1,151 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.fs.server.utils
+
+import com.google.common.cache.CacheBuilder
+import com.google.common.cache.CacheLoader
+import com.google.common.cache.LoadingCache
+import com.tencent.bkrepo.common.api.util.toJsonString
+import com.tencent.bkrepo.common.security.interceptor.devx.ApiAuth
+import com.tencent.bkrepo.common.security.interceptor.devx.DevXProperties
+import com.tencent.bkrepo.common.security.interceptor.devx.DevXWorkSpace
+import com.tencent.bkrepo.common.security.interceptor.devx.QueryResponse
+import com.tencent.bkrepo.fs.server.context.ReactiveRequestContextHolder
+import kotlinx.coroutines.reactor.mono
+import org.slf4j.LoggerFactory
+import org.springframework.http.HttpStatus
+import org.springframework.http.client.reactive.ReactorClientHttpConnector
+import org.springframework.web.reactive.function.client.ClientResponse
+import org.springframework.web.reactive.function.client.WebClient
+import org.springframework.web.reactive.function.client.awaitBody
+import reactor.core.publisher.Mono
+import reactor.netty.http.client.HttpClient
+import reactor.netty.http.client.PrematureCloseException
+import reactor.netty.resources.ConnectionProvider
+import reactor.util.retry.RetryBackoffSpec
+import java.time.Duration
+
+class DevxWorkspaceUtils(
+ devXProperties: DevXProperties
+) {
+
+ init {
+ Companion.devXProperties = devXProperties
+ }
+
+ companion object {
+ private val logger = LoggerFactory.getLogger(DevxWorkspaceUtils::class.java)
+ private lateinit var devXProperties: DevXProperties
+ private val httpClient by lazy {
+ val provider = ConnectionProvider.builder("DevX").maxIdleTime(Duration.ofSeconds(30L)).build()
+ val client = HttpClient.create(provider).responseTimeout(Duration.ofSeconds(30L))
+ val connector = ReactorClientHttpConnector(client)
+ WebClient.builder().clientConnector(connector).build()
+ }
+ private val projectIpsCache: LoadingCache>> by lazy { CacheBuilder.newBuilder()
+ .maximumSize(devXProperties.cacheSize)
+ .expireAfterWrite(devXProperties.cacheExpireTime)
+ .build(CacheLoader.from { key -> listIpFromProject(key) }) }
+
+ fun getIpList(projectId: String): Mono> {
+ return projectIpsCache.get(projectId)
+ }
+
+ suspend fun getWorkspace(): Mono {
+ val apiAuth = ApiAuth(devXProperties.appCode, devXProperties.appSecret)
+ val token = apiAuth.toJsonString().replace(System.lineSeparator(), "")
+ val workspaceUrl = devXProperties.workspaceUrl
+ val ip = ReactiveRequestContextHolder.getClientAddress()
+ return httpClient
+ .get()
+ .uri("$workspaceUrl?ip=$ip")
+ .header("X-Bkapi-Authorization", token)
+ .exchangeToMono {
+ mono { parseWorkSpaces(it).firstOrNull() }
+ }
+ .retryWhen(
+ RetryBackoffSpec
+ .backoff(2L, Duration.ofSeconds(1))
+ .filter {
+ val retry = it.cause is PrematureCloseException
+ logger.warn("request workspace of ip[$ip] failed, will retry: $retry")
+ retry
+ }
+ )
+ }
+
+ private fun listIpFromProject(projectId: String): Mono> {
+ val apiAuth = ApiAuth(devXProperties.appCode, devXProperties.appSecret)
+ val token = apiAuth.toJsonString().replace(System.lineSeparator(), "")
+ val workspaceUrl = devXProperties.workspaceUrl
+
+ logger.info("Update project[$projectId] ips.")
+ return httpClient
+ .get()
+ .uri("$workspaceUrl?project_id=$projectId")
+ .header("X-Bkapi-Authorization", token)
+ .exchangeToMono {
+ mono { parseResponse(it, projectId) }
+ }
+ .retryWhen(
+ RetryBackoffSpec
+ .backoff(2L, Duration.ofSeconds(1))
+ .filter {
+ val retry = it.cause is PrematureCloseException
+ logger.warn("request ips of project[$projectId] failed, will retry: $retry")
+ retry
+ }
+ )
+ }
+
+ private suspend fun parseResponse(response: ClientResponse, projectId: String): Set {
+ return if (response.statusCode() != HttpStatus.OK) {
+ val errorMsg = response.awaitBody()
+ logger.error("${response.statusCode()} $errorMsg")
+ devXProperties.projectCvmWhiteList[projectId] ?: emptySet()
+ } else {
+ val ips = HashSet()
+ devXProperties.projectCvmWhiteList[projectId]?.let { ips.addAll(it) }
+ response.awaitBody().data.forEach { workspace ->
+ workspace.innerIp?.substringAfter('.')?.let { ips.add(it) }
+ }
+ ips
+ }
+ }
+
+ private suspend fun parseWorkSpaces(response: ClientResponse): List {
+ return if (response.statusCode() != HttpStatus.OK) {
+ val errorMsg = response.awaitBody()
+ logger.error("${response.statusCode()} $errorMsg")
+ emptyList()
+ } else {
+ response.awaitBody().data
+ }
+ }
+ }
+}
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
index aee28270a7..0b72b245ef 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
@@ -224,7 +224,7 @@ class RepositoryServiceImpl(
Preconditions.checkArgument(checkInterceptorConfig(configuration), this::configuration.name)
// 确保项目一定存在
if (!projectService.checkExist(projectId)) {
- throw ErrorCodeException(ArtifactMessageCode.PROJECT_NOT_FOUND, name)
+ throw ErrorCodeException(ArtifactMessageCode.PROJECT_NOT_FOUND, projectId)
}
// 确保同名仓库不存在
if (checkExist(projectId, name)) {
From b7af0a63a9356f5730ed93c01f0ba31f8bbefa77 Mon Sep 17 00:00:00 2001
From: yaoxuwan
Date: Fri, 10 Nov 2023 15:18:28 +0800
Subject: [PATCH 03/74] =?UTF-8?q?feat:=20stat=E6=94=B9=E4=B8=BA=E5=8F=96no?=
=?UTF-8?q?de=E5=BF=AB=E7=85=A7=E6=95=B0=E6=8D=AE=20#1282=20(#1418)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: stat改为取node快照数据 #1282
* feat: stat改为取node快照数据 #1282
* feat: stat改为取node快照数据 #1282
* feat: stat改为取node快照数据 #1282
---
.../bkrepo/fs/server/api/RRepositoryClient.kt | 9 +++-
.../server/handler/NodeOperationsHandler.kt | 10 ++++-
.../bkrepo/repository/api/NodeClient.kt | 4 +-
.../bkrepo/repository/api/RepositoryClient.kt | 8 ++++
.../controller/service/NodeController.kt | 9 +++-
.../service/RepositoryController.kt | 5 +++
.../service/node/NodeStatsOperation.kt | 2 +-
.../service/node/impl/NodeServiceImpl.kt | 4 +-
.../service/node/impl/NodeStatsSupport.kt | 42 ++++++++++++++++++-
.../node/impl/edge/EdgeNodeServiceImpl.kt | 4 +-
.../service/repo/RepositoryService.kt | 3 ++
.../repo/impl/RepositoryServiceImpl.kt | 25 +++++++++++
.../CommitEdgeCenterRepositoryServiceImpl.kt | 10 ++++-
.../impl/edge/EdgeRepositoryServiceImpl.kt | 10 ++++-
.../repository/service/ServiceBaseTest.kt | 4 +-
15 files changed, 133 insertions(+), 16 deletions(-)
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt
index 68f01fd6c2..c7b5999857 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt
@@ -87,7 +87,8 @@ interface RRepositoryClient {
fun computeSize(
@PathVariable projectId: String,
@PathVariable repoName: String,
- @RequestParam fullPath: String
+ @RequestParam fullPath: String,
+ @RequestParam estimated: Boolean = false
): Mono>
@GetMapping("/repo/detail/{projectId}/{repoName}")
@@ -112,4 +113,10 @@ interface RRepositoryClient {
@PathVariable repoName: String,
@RequestParam fullPath: String
): Mono>>
+
+ @GetMapping("/repo/stat/{projectId}/{repoName}")
+ fun statRepo(
+ @PathVariable projectId: String,
+ @PathVariable repoName: String,
+ ): Mono>
}
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt
index c39e7118ab..128e0e4bc5 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt
@@ -55,9 +55,11 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest
import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest
import com.tencent.bkrepo.repository.pojo.node.service.NodeSetLengthRequest
import kotlinx.coroutines.reactor.awaitSingle
+import org.slf4j.LoggerFactory
import org.springframework.web.reactive.function.server.ServerRequest
import org.springframework.web.reactive.function.server.ServerResponse
import org.springframework.web.reactive.function.server.buildAndAwait
+import reactivefeign.client.ReadTimeoutException
import java.time.Duration
/**
@@ -153,7 +155,12 @@ class NodeOperationsHandler(
var res = statCache.getIfPresent(cacheKey)
if (res == null) {
val cap = ReactiveArtifactContextHolder.getRepoDetail().quota
- val nodeStat = rRepositoryClient.computeSize(projectId, repoName, fullPath).awaitSingle().data
+ val nodeStat = try {
+ rRepositoryClient.computeSize(projectId, repoName, fullPath, true).awaitSingle().data
+ } catch (e: ReadTimeoutException) {
+ logger.warn("get repo[$projectId/$repoName] stat timeout")
+ rRepositoryClient.statRepo(projectId, repoName).awaitSingle().data
+ }
res = StatResponse(
subNodeCount = nodeStat?.subNodeCount ?: UNKNOWN,
@@ -280,5 +287,6 @@ class NodeOperationsHandler(
companion object {
private const val UNKNOWN = -1L
+ private val logger = LoggerFactory.getLogger(NodeOperationsHandler::class.java)
}
}
diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt
index 5a25d11333..22854d0aad 100644
--- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt
+++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt
@@ -147,7 +147,9 @@ interface NodeClient {
@ApiParam(value = "仓库名称", required = true)
@PathVariable repoName: String,
@ApiParam(value = "节点完整路径", required = true)
- @RequestParam fullPath: String
+ @RequestParam fullPath: String,
+ @ApiParam(value = "估计值", required = false)
+ @RequestParam estimated: Boolean = false
): Response
@ApiOperation("查询文件节点数量")
diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt
index 482083acf9..3a6a797332 100644
--- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt
+++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt
@@ -34,6 +34,7 @@ package com.tencent.bkrepo.repository.api
import com.tencent.bkrepo.common.api.constant.REPOSITORY_SERVICE_NAME
import com.tencent.bkrepo.common.api.pojo.Page
import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo
import com.tencent.bkrepo.repository.pojo.project.RepoRangeQueryRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoDeleteRequest
@@ -128,4 +129,11 @@ interface RepositoryClient {
@PathVariable projectId: String,
@RequestBody option: RepoListOption
): Response>
+
+ @ApiOperation("查询仓库大小信息")
+ @GetMapping("/stat/{projectId}/{repoName}")
+ fun statRepo(
+ @PathVariable projectId: String,
+ @PathVariable repoName: String,
+ ): Response
}
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt
index c954cf223c..5b94e97a6d 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt
@@ -137,9 +137,14 @@ class NodeController(
)
}
- override fun computeSize(projectId: String, repoName: String, fullPath: String): Response {
+ override fun computeSize(
+ projectId: String,
+ repoName: String,
+ fullPath: String,
+ estimated: Boolean
+ ): Response {
val artifactInfo = DefaultArtifactInfo(projectId, repoName, fullPath)
- return ResponseBuilder.success(nodeService.computeSize(artifactInfo))
+ return ResponseBuilder.success(nodeService.computeSize(artifactInfo, estimated))
}
override fun countFileNode(projectId: String, repoName: String, path: String): Response {
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/RepositoryController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/RepositoryController.kt
index a025f37e90..5e02fb4b84 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/RepositoryController.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/RepositoryController.kt
@@ -35,6 +35,7 @@ import com.tencent.bkrepo.common.api.pojo.Page
import com.tencent.bkrepo.common.api.pojo.Response
import com.tencent.bkrepo.common.service.util.ResponseBuilder
import com.tencent.bkrepo.repository.api.RepositoryClient
+import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo
import com.tencent.bkrepo.repository.pojo.project.RepoRangeQueryRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoDeleteRequest
@@ -96,4 +97,8 @@ class RepositoryController(
): Response> {
return ResponseBuilder.success(repositoryService.listPermissionRepo(userId, projectId, option))
}
+
+ override fun statRepo(projectId: String, repoName: String): Response {
+ return ResponseBuilder.success(repositoryService.statRepo(projectId, repoName))
+ }
}
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeStatsOperation.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeStatsOperation.kt
index 92e3021f91..587c19d8b3 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeStatsOperation.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeStatsOperation.kt
@@ -43,7 +43,7 @@ interface NodeStatsOperation {
/**
* 计算文件或者文件夹大小
*/
- fun computeSize(artifact: ArtifactInfo): NodeSizeInfo
+ fun computeSize(artifact: ArtifactInfo, estimated: Boolean = false): NodeSizeInfo
/**
* 查询文件节点数量
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt
index 07aee69c3c..1d3f738f21 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt
@@ -75,8 +75,8 @@ class NodeServiceImpl(
messageSupplier
) {
- override fun computeSize(artifact: ArtifactInfo): NodeSizeInfo {
- return NodeStatsSupport(this).computeSize(artifact)
+ override fun computeSize(artifact: ArtifactInfo, estimated: Boolean): NodeSizeInfo {
+ return NodeStatsSupport(this).computeSize(artifact, estimated)
}
override fun aggregateComputeSize(criteria: Criteria): Long {
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeStatsSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeStatsSupport.kt
index ebf637a5b4..7fd52a4db1 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeStatsSupport.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeStatsSupport.kt
@@ -31,6 +31,7 @@
package com.tencent.bkrepo.repository.service.node.impl
+import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.api.exception.ErrorCodeException
import com.tencent.bkrepo.common.artifact.api.ArtifactInfo
import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode
@@ -55,7 +56,7 @@ open class NodeStatsSupport(
private val nodeDao: NodeDao = nodeBaseService.nodeDao
- override fun computeSize(artifact: ArtifactInfo): NodeSizeInfo {
+ override fun computeSize(artifact: ArtifactInfo, estimated: Boolean): NodeSizeInfo {
val projectId = artifact.projectId
val repoName = artifact.repoName
val fullPath = artifact.getArtifactFullPath()
@@ -65,6 +66,9 @@ open class NodeStatsSupport(
if (!node.folder) {
return NodeSizeInfo(subNodeCount = 0, subNodeWithoutFolderCount = 0, size = node.size)
}
+ if (estimated) {
+ return computeEstimatedSize(node)
+ }
val listOption = NodeListOption(includeFolder = true, deep = true)
val criteria = NodeQueryHelper.nodeListCriteria(projectId, repoName, node.fullPath, listOption)
val count = nodeDao.count(Query(criteria))
@@ -84,6 +88,42 @@ open class NodeStatsSupport(
return NodeSizeInfo(subNodeCount = count, subNodeWithoutFolderCount = countWithOutFolder, size = size)
}
+ /**
+ * 计算目录大小信息的估计值
+ *
+ */
+ private fun computeEstimatedSize(node: TNode): NodeSizeInfo {
+ val countCriteria = NodeQueryHelper.nodeListCriteria(
+ projectId = node.projectId,
+ repoName = node.repoName,
+ path = node.fullPath,
+ option = NodeListOption(includeFolder = true, deep = true)
+ )
+ val count = nodeDao.count(Query(countCriteria))
+ if (node.fullPath != StringPool.ROOT) {
+ return NodeSizeInfo(count, node.nodeNum ?: 0, node.size)
+ }
+ val criteria = NodeQueryHelper.nodeListCriteria(
+ projectId = node.projectId,
+ repoName = node.repoName,
+ path = node.fullPath,
+ option = NodeListOption(includeFolder = true, deep = false)
+ )
+
+ val aggregation = newAggregation(
+ match(criteria),
+ group().sum(TNode::size.name).`as`(NodeSizeInfo::size.name)
+ .sum(TNode::nodeNum.name).`as`(NodeSizeInfo::subNodeCount.name)
+ )
+ val aggregateResult = nodeDao.aggregate(aggregation, HashMap::class.java)
+ val data = aggregateResult.mappedResults.firstOrNull()
+ return NodeSizeInfo(
+ subNodeCount = count,
+ subNodeWithoutFolderCount = data?.get(NodeSizeInfo::subNodeCount.name) as? Long ?: 0,
+ size = data?.get(NodeSizeInfo::size.name) as? Long ?: 0
+ )
+ }
+
override fun countFileNode(artifact: ArtifactInfo): Long {
with(artifact) {
val listOption = NodeListOption(
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt
index d8a46c4464..e3144d056a 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt
@@ -83,8 +83,8 @@ class EdgeNodeServiceImpl(
messageSupplier,
clusterProperties
) {
- override fun computeSize(artifact: ArtifactInfo): NodeSizeInfo {
- return NodeStatsSupport(this).computeSize(artifact)
+ override fun computeSize(artifact: ArtifactInfo, estimated: Boolean): NodeSizeInfo {
+ return NodeStatsSupport(this).computeSize(artifact, estimated)
}
override fun aggregateComputeSize(criteria: Criteria): Long {
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/RepositoryService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/RepositoryService.kt
index 8ecfb9e2d0..68967bd46e 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/RepositoryService.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/RepositoryService.kt
@@ -29,6 +29,7 @@ package com.tencent.bkrepo.repository.service.repo
import com.tencent.bkrepo.common.api.pojo.Page
import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
+import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo
import com.tencent.bkrepo.repository.pojo.project.RepoRangeQueryRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoDeleteRequest
@@ -163,4 +164,6 @@ interface RepositoryService {
fun allRepos(projectId: String?, repoName: String?, repoType: RepositoryType?): List
+
+ fun statRepo(projectId: String, repoName: String): NodeSizeInfo
}
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
index 0b72b245ef..736c78e63f 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
@@ -28,6 +28,7 @@
package com.tencent.bkrepo.repository.service.repo.impl
import com.tencent.bkrepo.auth.api.ServicePermissionClient
+import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.api.exception.ErrorCodeException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.api.pojo.Page
@@ -57,8 +58,12 @@ import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publi
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier
import com.tencent.bkrepo.repository.config.RepositoryProperties
+import com.tencent.bkrepo.repository.dao.NodeDao
import com.tencent.bkrepo.repository.dao.RepositoryDao
+import com.tencent.bkrepo.repository.dao.repository.ProjectMetricsRepository
import com.tencent.bkrepo.repository.model.TRepository
+import com.tencent.bkrepo.repository.pojo.node.NodeListOption
+import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo
import com.tencent.bkrepo.repository.pojo.project.RepoRangeQueryRequest
import com.tencent.bkrepo.repository.pojo.proxy.ProxyChannelCreateRequest
import com.tencent.bkrepo.repository.pojo.proxy.ProxyChannelDeleteRequest
@@ -75,6 +80,7 @@ import com.tencent.bkrepo.repository.service.repo.ProjectService
import com.tencent.bkrepo.repository.service.repo.ProxyChannelService
import com.tencent.bkrepo.repository.service.repo.RepositoryService
import com.tencent.bkrepo.repository.service.repo.StorageCredentialService
+import com.tencent.bkrepo.repository.util.NodeQueryHelper
import com.tencent.bkrepo.repository.util.RepoEventFactory.buildCreatedEvent
import com.tencent.bkrepo.repository.util.RepoEventFactory.buildDeletedEvent
import com.tencent.bkrepo.repository.util.RepoEventFactory.buildUpdatedEvent
@@ -110,6 +116,8 @@ class RepositoryServiceImpl(
private val repositoryProperties: RepositoryProperties,
private val messageSupplier: MessageSupplier,
private val servicePermissionClient: ServicePermissionClient,
+ private val projectMetricsRepository: ProjectMetricsRepository,
+ private val nodeDao: NodeDao
) : RepositoryService {
init {
@@ -368,6 +376,23 @@ class RepositoryServiceImpl(
return result.map { convertToInfo(it) }
}
+ override fun statRepo(projectId: String, repoName: String): NodeSizeInfo {
+ val query = Query(NodeQueryHelper.nodeListCriteria(
+ projectId = projectId,
+ repoName = repoName,
+ path = StringPool.ROOT,
+ option = NodeListOption(includeFolder = true, deep = true)
+ ))
+ val count = nodeDao.count(query)
+ val projectMetrics = projectMetricsRepository.findFirstByProjectIdOrderByCreatedDateDesc(projectId)
+ val repoMetrics = projectMetrics?.repoMetrics?.firstOrNull { it.repoName == repoName }
+ return NodeSizeInfo(
+ subNodeCount = count,
+ subNodeWithoutFolderCount = repoMetrics?.num ?: 0,
+ size = repoMetrics?.size ?: 0
+ )
+ }
+
/**
* 获取仓库下的代理地址信息
*/
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/center/CommitEdgeCenterRepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/center/CommitEdgeCenterRepositoryServiceImpl.kt
index b7ba7e077e..e423817588 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/center/CommitEdgeCenterRepositoryServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/center/CommitEdgeCenterRepositoryServiceImpl.kt
@@ -45,7 +45,9 @@ import com.tencent.bkrepo.common.service.cluster.CommitEdgeCenterCondition
import com.tencent.bkrepo.common.service.util.SpringContextUtils
import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier
import com.tencent.bkrepo.repository.config.RepositoryProperties
+import com.tencent.bkrepo.repository.dao.NodeDao
import com.tencent.bkrepo.repository.dao.RepositoryDao
+import com.tencent.bkrepo.repository.dao.repository.ProjectMetricsRepository
import com.tencent.bkrepo.repository.model.TRepository
import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoDeleteRequest
@@ -72,7 +74,9 @@ class CommitEdgeCenterRepositoryServiceImpl(
private val repositoryProperties: RepositoryProperties,
messageSupplier: MessageSupplier,
servicePermissionClient: ServicePermissionClient,
- private val clusterProperties: ClusterProperties
+ private val clusterProperties: ClusterProperties,
+ projectMetricsRepository: ProjectMetricsRepository,
+ nodeDao: NodeDao
) : RepositoryServiceImpl(
repositoryDao,
nodeService,
@@ -81,7 +85,9 @@ class CommitEdgeCenterRepositoryServiceImpl(
proxyChannelService,
repositoryProperties,
messageSupplier,
- servicePermissionClient
+ servicePermissionClient,
+ projectMetricsRepository,
+ nodeDao
) {
override fun determineStorageKey(request: RepoCreateRequest): String? {
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/edge/EdgeRepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/edge/EdgeRepositoryServiceImpl.kt
index de1ac9d571..de898d98f3 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/edge/EdgeRepositoryServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/edge/EdgeRepositoryServiceImpl.kt
@@ -36,7 +36,9 @@ import com.tencent.bkrepo.common.service.feign.FeignClientFactory
import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier
import com.tencent.bkrepo.repository.api.cluster.ClusterRepositoryClient
import com.tencent.bkrepo.repository.config.RepositoryProperties
+import com.tencent.bkrepo.repository.dao.NodeDao
import com.tencent.bkrepo.repository.dao.RepositoryDao
+import com.tencent.bkrepo.repository.dao.repository.ProjectMetricsRepository
import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoDeleteRequest
import com.tencent.bkrepo.repository.pojo.repo.RepoUpdateRequest
@@ -60,7 +62,9 @@ class EdgeRepositoryServiceImpl(
repositoryProperties: RepositoryProperties,
messageSupplier: MessageSupplier,
servicePermissionClient: ServicePermissionClient,
- clusterProperties: ClusterProperties
+ clusterProperties: ClusterProperties,
+ projectMetricsRepository: ProjectMetricsRepository,
+ nodeDao: NodeDao
) : RepositoryServiceImpl(
repositoryDao,
nodeService,
@@ -69,7 +73,9 @@ class EdgeRepositoryServiceImpl(
proxyChannelService,
repositoryProperties,
messageSupplier,
- servicePermissionClient
+ servicePermissionClient,
+ projectMetricsRepository,
+ nodeDao
) {
private val centerRepoClient: ClusterRepositoryClient by lazy {
diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt
index 6cdb17e5ad..ead867e35b 100644
--- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt
+++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt
@@ -54,6 +54,7 @@ import com.tencent.bkrepo.repository.UT_REPO_DISPLAY
import com.tencent.bkrepo.repository.UT_REPO_NAME
import com.tencent.bkrepo.repository.UT_USER
import com.tencent.bkrepo.repository.config.RepositoryProperties
+import com.tencent.bkrepo.repository.dao.NodeDao
import com.tencent.bkrepo.repository.dao.ProjectDao
import com.tencent.bkrepo.repository.dao.ProxyChannelDao
import com.tencent.bkrepo.repository.dao.RepositoryDao
@@ -88,7 +89,8 @@ import org.springframework.test.context.TestPropertySource
RepositoryDao::class,
ProxyChannelDao::class,
HttpAuthProperties::class,
- SpringContextUtils::class
+ SpringContextUtils::class,
+ NodeDao::class
)
@ComponentScan("com.tencent.bkrepo.repository.service")
@TestPropertySource(locations = ["classpath:bootstrap-ut.properties", "classpath:center-ut.properties"])
From d7d299b4009394accd96206d15fee792d2391686 Mon Sep 17 00:00:00 2001
From: gujunling <34835140+gujunling@users.noreply.github.com>
Date: Mon, 13 Nov 2023 11:58:43 +0800
Subject: [PATCH 04/74] =?UTF-8?q?feat:=20=E5=9B=BD=E9=99=85=E5=8C=96?=
=?UTF-8?q?=E9=81=97=E6=BC=8F=E4=B9=8B=E5=A4=84=E5=AE=8C=E5=96=84=20#1410?=
=?UTF-8?q?=20(#1420)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: 依赖源仓库的晋级操作的英文翻译统一使用upgrade #1410
* feat: 仓库的英文翻译不准确，应该是repository #1410
* feat: 制品分析的扫描记录详情页的质量规则显示在英文状态下且数据量大的时候会出现部分换行，修改为一行一个 #1410
* feat: 制品分析的总览视图翻译不准确，且不太合适，修改为制品视图，英文翻译为 Artifact view #1410
* feat: 制品分发的分发计划列表、日志弹窗、日志详情页等页面的状态英文翻译不统一，统一修改为 Success、Fail及页面样式调整 #1410
---
.../src/views/planManage/index.vue | 18 ++++-----
.../src/views/planManage/logDetail.vue | 4 +-
.../src/views/planManage/planLog.vue | 6 +--
.../views/repoCommon/commonVersionDetail.vue | 2 +-
.../repoScan/artiReport/artifactInfo.vue | 2 +-
src/frontend/locale/repository/en-US.json | 37 +++++++++----------
src/frontend/locale/repository/zh-CN.json | 5 +--
7 files changed, 36 insertions(+), 38 deletions(-)
diff --git a/src/frontend/devops-repository/src/views/planManage/index.vue b/src/frontend/devops-repository/src/views/planManage/index.vue
index adc5e2a039..47d7385bdd 100644
--- a/src/frontend/devops-repository/src/views/planManage/index.vue
+++ b/src/frontend/devops-repository/src/views/planManage/index.vue
@@ -46,18 +46,18 @@
{{row.name}}
-
+
+ {{ row.remoteClusters.map(v => v.name).join('、') }}
+
+
{{ { 'REPOSITORY': $t('synchronizeRepository'), 'PACKAGE': $t('synchronizePackage'), 'PATH': $t('synchronizePath') }[row.replicaObjectType] }}
-
- {{ row.remoteClusters.map(v => v.name).join('、') }}
-
-
+
{{ getExecutionStrategy(row) }}
-
+
{{formatDate(row.lastExecutionTime)}}
@@ -65,12 +65,12 @@
{{row.lastExecutionStatus ? $t(`asyncPlanStatusEnum.${row.lastExecutionStatus}`) : $t('notExecuted')}}
-
+
-
+
-
+
-
-
+
+
{{ $t('total') }}
- {{ $t('success') }}
- {{ $t('fail') }}
+ {{ $t('asyncPlanStatusEnum.SUCCESS') }}
+ {{ $t('asyncPlanStatusEnum.FAILED') }}
-
+ {{$t(`asyncPlanStatusEnum.${row.status}`) || $t('notExecuted')}}
diff --git a/src/frontend/devops-repository/src/views/repoCommon/commonVersionDetail.vue b/src/frontend/devops-repository/src/views/repoCommon/commonVersionDetail.vue
index 7c083c63d7..0b8b9156f4 100644
--- a/src/frontend/devops-repository/src/views/repoCommon/commonVersionDetail.vue
+++ b/src/frontend/devops-repository/src/views/repoCommon/commonVersionDetail.vue
@@ -236,7 +236,7 @@
return [
...(!metadataMap.forbidStatus
? [
- this.permission.edit && { clickEvent: () => this.$emit('tag'), label: this.$t('promotion'), disabled: (basic.stageTag || '').includes('@release') },
+ this.permission.edit && { clickEvent: () => this.$emit('tag'), label: this.$t('upgrade'), disabled: (basic.stageTag || '').includes('@release') },
this.showRepoScan && { clickEvent: () => this.$emit('scan'), label: this.$t('scanArtifact') }
]
: []),
diff --git a/src/frontend/devops-repository/src/views/repoScan/artiReport/artifactInfo.vue b/src/frontend/devops-repository/src/views/repoScan/artiReport/artifactInfo.vue
index f7743d9e59..0645499f75 100644
--- a/src/frontend/devops-repository/src/views/repoScan/artiReport/artifactInfo.vue
+++ b/src/frontend/devops-repository/src/views/repoScan/artiReport/artifactInfo.vue
@@ -130,7 +130,7 @@
}
}
.arti-leak {
- grid-template: auto / repeat(2, 1fr);
+ grid-template: auto / repeat(1, 1fr);
}
}
diff --git a/src/frontend/locale/repository/en-US.json b/src/frontend/locale/repository/en-US.json
index c0d3595782..b3c121f27c 100644
--- a/src/frontend/locale/repository/en-US.json
+++ b/src/frontend/locale/repository/en-US.json
@@ -24,7 +24,7 @@
"repoBaseInfo": "Base Info",
"proxyConfig": "Proxy Config",
"repoAddress": "Address",
- "proxyConfigTips": "When pulling a artifact that does not exist in this warehouse, it will try to pull it from the configured proxy address. Priority: top to bottom.",
+ "proxyConfigTips": "When pulling a artifact that does not exist in this repository, it will try to pull it from the configured proxy address. Priority: top to bottom.",
"addProxy": "Add Proxy",
"editProxy": "Edit Proxy",
"sameProxyExist": "A proxy source with the same name exists",
@@ -33,7 +33,7 @@
"fileDownloadError": "The artifact you are downloading does not comply with the download rules, please contact the ({0}) project administrator",
"origin": "Origin",
"address": "Address",
- "operation": "Actions",
+ "operation": "Operational",
"publicProxy": "Public proxy",
"privateProxy": "Private proxy",
"allTypes": "All types",
@@ -94,7 +94,7 @@
"copy": "Copy",
"move": "Move",
"detail": "Detail",
- "upgrade": "upgrade",
+ "upgrade": "Upgrade",
"baseInfo": "Base Info",
"metaData": "Meta Data",
"key": "Key",
@@ -223,7 +223,7 @@
"dockerRepoTip": "The docker repository name does not support capital English letters",
"openPublicLabel": "Anonymous Download Supported",
"openPublicTip": "No authentication, any terminal can download",
- "repoNamePlaceHolder": "Please enter the warehouse name, press enter to search",
+ "repoNamePlaceHolder": "Please enter the repository name, press enter to search",
"built-in": "Built-in",
"system": "System",
"public": "Public",
@@ -248,7 +248,6 @@
"versionPlaceHolder": "Please enter the version, press Enter to search",
"liftBan": "Enable",
"forbiddenUse": "Disable",
- "promotion": "Promotion",
"associatedUser": "Entity user",
"normalUser": "Normal user",
"totalVersionCount": "total of {0} versions",
@@ -437,7 +436,7 @@
"endTime":"End time",
"scanArtifactNum":"Artifacts scanned",
"scanCompletionTime":"Scan completion time",
- "overview":"Overview",
+ "overview":"Artifact view",
"taskView":"Task view",
"stopScanSuccessMsg": "Aborted scan successfully",
"confirmStopScanMsg":"Confirm to abort the scan task",
@@ -476,7 +475,7 @@
"userPlaceHolder": "Please enter the account number/Chinese name, press Enter to search",
"artifactPlaceHolder":"Please enter the artifact name, press Enter to search",
"setCredentials":"Set credentials",
- "dockerGuideSubTitle":"Execute the following command on the command line to log in to the warehouse",
+ "dockerGuideSubTitle":"Execute the following command on the command line to log in to the repository",
"push":"Push",
"dockerPushGuideSubTitle1":"1. Execute the following command on the command line to label the local image",
"dockerPushGuideSubTitle2":"2. Execute the following command on the command line to push",
@@ -489,9 +488,9 @@
"npmCreditGuideSubTitle4":"2. After copying to the command line window, press Enter",
"npmCreditGuideSubTitle5":"3. Copy the encoded token and replace ",
"npmCreditGuideSubTitle6":"Method 2: Set Credentials Using the Interactive Command Line",
- "npmCreditGuideSubTitle7":"Using the command line window, set npm registry as the current artifact library warehouse",
+ "npmCreditGuideSubTitle7":"Using the command line window, set npm registry as the current artifact library repository",
"npmCreditGuideSubTitle8":"Enter the command line window to log in according to user credentials",
- "emptyGuideProxyMsg":"To represent artifact from other warehouses",
+ "emptyGuideProxyMsg":"To represent artifact from other repositories",
"pushGuideSubTitle":"Execute the following command on the command line to push the artifact",
"rpmPullGuideSunTitle1":"Pull using the rpm command",
"rpmPullGuideSunTitle2":"Pull using the yum command",
@@ -504,14 +503,14 @@
"pypiPushGuideSubTitle":"Enter the Python project directory, execute the following command on the command line to push",
"cmdPullGuideSubTitle":"Execute the following command on the command line to pull",
"pypiInstallGuideSubTitle":"Pull the artifact by specifying the registry",
- "composerCreditGuideSubTitle1":"1. In the file directory of the Composer artifact, use the command line to execute the following command to configure the artifact warehouse credentials",
- "composerCreditGuideSubTitle2":"2. Add auth.json to the file directory of Composer artifacts to configure warehouse authentication information",
+ "composerCreditGuideSubTitle1":"1. In the file directory of the Composer artifact, use the command line to execute the following command to configure the artifact repository credentials",
+ "composerCreditGuideSubTitle2":"2. Add auth.json to the file directory of Composer artifacts to configure repository authentication information",
"composerPushGuideSubTitle":"Upload the tarball to the repository using the cURL command",
- "nugetCreditGuideSubTitle":"Execute the following command on the command line to configure artifact warehouse credentials:",
+ "nugetCreditGuideSubTitle":"Execute the following command on the command line to configure artifact repository credentials:",
"nugetPushGuideSubTitle":"Replace with the name of the local artifact, and execute the following command on the command line to push the artifact:",
"nugetPullGuideSubTitle":"Execute the following commands on the command line to pull artifacts:",
"nugetDeleteGuideSubTitle":"Note: Artifacts deleted by this operation cannot be restored",
- "npmDownloadGuideSubTitle1":"1. After setting the warehouse address, you can use the following command to pull the artifact",
+ "npmDownloadGuideSubTitle1":"1. After setting the repository address, you can use the following command to pull the artifact",
"npmDownloadGuideSubTitle2":"2. You can also pull the artifact by specifying the registry, as follows",
"mavenCreditGuideSubTitle1":"Use xml: set the account password in the configuration file conf/settings.xml; settings.xml in the project can also be set, high priority",
"mavenCreditGuideSubTitle2":"Using Groovy DSL/Kotlin DSL: Add the following configuration to the gradle.properties file in the same directory as the project file build.gradle",
@@ -530,7 +529,7 @@
"mavenPullGuideSubTitle4":"Using Groovy DSL: Execute the following command on the command line to pull artifacts",
"mavenPullGuideSubTitle5":"Using Kotlin DSL: Add the following configuration to your project's build.gradle file",
"mavenPullGuideSubTitle6":"Using Kotlin DSL: Execute the following command on the command line to pull artifacts",
- "helmCreditGuideSubTitle1":"1. Execute the following command on the command line to configure artifact warehouse credentials",
+ "helmCreditGuideSubTitle1":"1. Execute the following command on the command line to configure artifact repository credentials",
"helmCreditGuideSubTitle2":"2. Update local repo information",
"helmPushGuideSubTitle1":"Push Chart using cURL command",
"helmPushGuideSubTitle2":"Push Chart Provenance using cURL command",
@@ -608,7 +607,7 @@
"asyncPlanStatusEnum": {
"RUNNING": "Running",
"SUCCESS": "Success",
- "FAILED": "Failed"
+ "FAILED": "Fail"
},
"conflictStrategy":"Conflict Strategy",
"syncObject":"Object",
@@ -619,9 +618,9 @@
"terminateSyncLabel":"Terminate",
"terminateSyncTip":"When the same artifact exists on the target node, terminate the execution plan",
"artifacts":"Artifacts",
- "repoTip":"Sync multiple warehouses",
- "artifactTip":"Sync multiple artifacts under the same warehouse",
- "pathTip": "Sync multiple files under the same warehouse",
+ "repoTip":"Sync multiple repositories",
+ "artifactTip":"Sync multiple artifacts under the same repository",
+ "pathTip": "Sync multiple files under the same repository",
"time":"Time",
"validateTimeRule":"Current time has expired",
"cronMsg":"cron",
@@ -646,7 +645,7 @@
"startExecutionTime":"Starting time",
"endExecutionTime":"End Time",
"planLogTitle":"{name} execution log",
- "usedQuotaCapacity":"Quota capacity used by the warehouse",
+ "usedQuotaCapacity":"Quota capacity used by the repository",
"nodeTypeEnum": {
"CENTER": "center",
"EDGE": "edge",
diff --git a/src/frontend/locale/repository/zh-CN.json b/src/frontend/locale/repository/zh-CN.json
index 0792c18536..61e80e3c3f 100644
--- a/src/frontend/locale/repository/zh-CN.json
+++ b/src/frontend/locale/repository/zh-CN.json
@@ -261,7 +261,6 @@
"versionPlaceHolder": "请输入版本, 按Enter键搜索",
"liftBan": "解除禁止",
"forbiddenUse": "禁止使用",
- "promotion": "晋级",
"associatedUser": "关联用户",
"normalUser": "普通用户",
"totalVersionCount": "共计{0}个版本",
@@ -450,7 +449,7 @@
"endTime":"结束时间",
"scanArtifactNum":"扫描制品数",
"scanCompletionTime":"扫描完成时间",
- "overview":"总览视图",
+ "overview":"制品视图",
"taskView":"任务视图",
"stopScanSuccessMsg": "中止扫描成功",
"confirmStopScanMsg":"确认中止扫描任务",
@@ -620,7 +619,7 @@
"stopPlanSuccess": "停用计划成功",
"asyncPlanStatusEnum": {
"RUNNING": "进行中",
- "SUCCESS": "已完成",
+ "SUCCESS": "同步成功",
"FAILED": "同步异常"
},
"conflictStrategy":"冲突策略",
From c4613b6cd4421e9734513f4cf82836e81e5ad95d Mon Sep 17 00:00:00 2001
From: gujunling <34835140+gujunling@users.noreply.github.com>
Date: Mon, 13 Nov 2023 11:59:05 +0800
Subject: [PATCH 05/74] =?UTF-8?q?style:=20=E5=88=B6=E5=93=81=E4=BB=93?=
=?UTF-8?q?=E5=BA=93=E5=88=97=E8=A1=A8=E4=B8=AD=E4=BB=93=E5=BA=93=E6=A0=87?=
=?UTF-8?q?=E7=AD=BE=E6=B2=A1=E6=9C=89=E5=9E=82=E7=9B=B4=E5=B1=85=E4=B8=AD?=
=?UTF-8?q?=E6=98=BE=E7=A4=BA=20#1421=20(#1422)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
src/frontend/devops-repository/src/scss/index.scss | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/frontend/devops-repository/src/scss/index.scss b/src/frontend/devops-repository/src/scss/index.scss
index d684b2aee9..d84f736bd3 100644
--- a/src/frontend/devops-repository/src/scss/index.scss
+++ b/src/frontend/devops-repository/src/scss/index.scss
@@ -136,6 +136,7 @@ body {
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
+ vertical-align: middle;
&:before {
content: attr(data-name);
}
From efc76a8c851c1eec5c13079df2f0724317bc8357 Mon Sep 17 00:00:00 2001
From: zacYL <100330102+zacYL@users.noreply.github.com>
Date: Mon, 13 Nov 2023 11:59:42 +0800
Subject: [PATCH 06/74] =?UTF-8?q?feat:=20=E5=88=A0=E9=99=A4=E9=85=8D?=
=?UTF-8?q?=E7=BD=AE=E7=9A=84=E7=9B=AE=E5=BD=95=E7=9A=84=E5=AD=90=E7=9B=AE?=
=?UTF-8?q?=E5=BD=95=E4=B8=8B=E7=9A=84=E8=8A=82=E7=82=B9=20#1425=20(#1428)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../kotlin/com/tencent/bkrepo/job/batch/ArtifactCleanupJob.kt | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ArtifactCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ArtifactCleanupJob.kt
index 8c65dbe34f..8a9a76aaa6 100644
--- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ArtifactCleanupJob.kt
+++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ArtifactCleanupJob.kt
@@ -62,6 +62,7 @@ import org.springframework.stereotype.Component
import java.time.Duration
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
+import java.util.regex.Pattern
/**
* 根据仓库配置的清理策略清理对应仓库下的制品
@@ -172,7 +173,7 @@ class ArtifactCleanupJob(
.and(LAST_MODIFIED_DATE).lt(cleanupDate).and(ID).gt(lastId)
.apply {
if (!cleanupFolders.isNullOrEmpty()) {
- this.and(PATH).`in`(cleanupFolders.map { toPath(it) })
+ this.and(PATH).`in`(cleanupFolders.map { Pattern.compile("^${toPath(it)}") })
}
}
).limit(BATCH_SIZE)
From 09decfcf83a6623cf67efade8e480824d96c4960 Mon Sep 17 00:00:00 2001
From: yaoxuwan
Date: Mon, 13 Nov 2023 12:00:11 +0800
Subject: [PATCH 07/74] =?UTF-8?q?feat:=20=E8=A1=A5=E5=85=85=E5=88=9B?=
=?UTF-8?q?=E5=BB=BA=E4=BB=93=E5=BA=93=E3=80=81=E4=BF=AE=E6=94=B9=E4=BB=93?=
=?UTF-8?q?=E5=BA=93=E5=B1=9E=E6=80=A7=E7=9B=B8=E5=85=B3=E7=9A=84=E5=AD=97?=
=?UTF-8?q?=E6=AE=B5=E8=AF=B4=E6=98=8E=20#1267=20(#1429)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
docs/apidoc/repo/repository.md | 43 ++++++++++++++++++++++++++++++++++
1 file changed, 43 insertions(+)
diff --git a/docs/apidoc/repo/repository.md b/docs/apidoc/repo/repository.md
index 6b552903e0..783aa2b067 100644
--- a/docs/apidoc/repo/repository.md
+++ b/docs/apidoc/repo/repository.md
@@ -597,6 +597,49 @@
|username|string|否|无|代理源认证用户名|channel username|
|password|string|否|无|代理源认证密码|channel password|
+### generic仓库下载限制配置项
+```json
+"configuration" : {
+ "settings" : {
+ "interceptors" : [ {
+ "type" : "MOBILE",
+ "rules" : {
+ "filename" : "*.apk",
+ "metadata" : "key:value"
+ }
+ }, {
+ "type" : "WEB",
+ "rules" : {
+ "filename" : "*.txt",
+ "metadata" : "key:value"
+ }
+ }, {
+ "type" : "IP_SEGMENT",
+ "rules" : {
+ "ipSegment" : [ "10.0.0.0/24", "11.1.0.0/16" ],
+ "whitelistUser" : [ "userId" ],
+ "officeNetwork" : false
+ }
+ } ]
+ }
+ },
+```
+|字段|类型|是否必须|默认值|说明|Description|
+|---|---|---|---|---|---|
+|type|String|是|无|下载限制类型，MOBILE移动端下载限制，WEB网页端下载限制，IP_SEGMENT IP段下载限制|download interceptor type|
+|rules|Map|是|无|下载限制规则，符合规则的可以下载|download interceptor rules|
+
+- **移动端下载限制**
+ - filename 文件名规则，支持通配符*
+ - metadata 元数据规则，格式为key:value
+- **网页端下载限制**
+ - filename 文件名规则，支持通配符*
+ - metadata 元数据规则，格式为key:value
+- **IP段下载限制**
+ - ipSegmnt Ip段规则，支持多个Ip段
+ - officeNetwork 办公网下载规则，办公网网段由后台配置的，开启后ip段添加办公网网段
+ - whitelistUser 白名单用户，不受Ip段下载限制约束
+
### 依赖源的差异化配置项
各个依赖源的差异化配置通过`settings`进行配置，每项配置的具体含义请参考依赖源文档。
\ No newline at end of file
From cc539db7a0f8166ad98aebb082b70e5fd7677dc1 Mon Sep 17 00:00:00 2001
From: kunlongli <16629885+cnlkl@users.noreply.github.com>
Date: Tue, 14 Nov 2023 14:37:05 +0800
Subject: [PATCH 08/74] =?UTF-8?q?bug:=20=E4=BF=AE=E5=A4=8DExceptionHandler?=
=?UTF-8?q?=E6=8A=A5HttpMediaTypeNotAcceptableException=20#1423=20(#1433)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../bkrepo/common/api/constant/MediaTypes.kt | 1 +
.../exception/AbstractExceptionHandler.kt | 18 ++++++++++++++++++
2 files changed, 19 insertions(+)
diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/MediaTypes.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/MediaTypes.kt
index a349c7ac19..9163c5e3ae 100644
--- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/MediaTypes.kt
+++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/MediaTypes.kt
@@ -36,6 +36,7 @@ object MediaTypes {
const val APPLICATION_JSON = "application/json; charset=utf-8"
const val APPLICATION_XML = "application/xml;charset=utf-8"
const val APPLICATION_JSON_WITHOUT_CHARSET = "application/json"
+ const val APPLICATION_XML_WITHOUT_CHARSET = "application/xml"
const val TEXT_HTML = "text/html; charset=utf-8"
const val TEXT_PLAIN = "text/plain"
const val APPLICATION_YAML = "application/x-yaml"
diff --git a/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/AbstractExceptionHandler.kt b/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/AbstractExceptionHandler.kt
index b975dd0131..eb342bc0d6 100644
--- a/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/AbstractExceptionHandler.kt
+++ b/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/AbstractExceptionHandler.kt
@@ -31,7 +31,9 @@
package com.tencent.bkrepo.common.service.exception
+import com.tencent.bkrepo.common.api.constant.HttpHeaders
import com.tencent.bkrepo.common.api.constant.HttpStatus
+import com.tencent.bkrepo.common.api.constant.MediaTypes
import com.tencent.bkrepo.common.api.exception.ErrorCodeException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.api.pojo.Response
@@ -50,6 +52,7 @@ open class AbstractExceptionHandler {
val errorMessage = LocaleMessageUtils.getLocalizedMessage(exception.messageCode, exception.params)
LoggerHolder.logErrorCodeException(exception, "[${exception.messageCode.getCode()}]$errorMessage")
HttpContextHolder.getResponse().status = exception.status.value
+ updateContentType()
return ResponseBuilder.fail(exception.messageCode.getCode(), errorMessage)
}
@@ -63,6 +66,21 @@ open class AbstractExceptionHandler {
val code = CommonMessageCode.SYSTEM_ERROR.getCode()
LoggerHolder.logException(exception, "[$code]${exception.message}", true)
HttpContextHolder.getResponse().status = HttpStatus.INTERNAL_SERVER_ERROR.value
+ updateContentType()
return ResponseBuilder.fail(code, errorMessage)
}
+
+ protected fun updateContentType(contentType: String? = null) {
+ if (contentType != null) {
+ HttpContextHolder.getResponse().contentType = contentType
+ return
+ }
+ val accept = HttpContextHolder.getRequest().getHeader(HttpHeaders.ACCEPT)
+ if (HttpContextHolder.getResponse().contentType == null &&
+ !accept.startsWith(MediaTypes.APPLICATION_JSON_WITHOUT_CHARSET, true) &&
+ !accept.startsWith(MediaTypes.APPLICATION_XML_WITHOUT_CHARSET, true)
+ ) {
+ HttpContextHolder.getResponse().contentType = MediaTypes.APPLICATION_JSON
+ }
+ }
}
From 1f3d9cae63483af8147e963b8619ba0abd9726af Mon Sep 17 00:00:00 2001
From: kunlongli <16629885+cnlkl@users.noreply.github.com>
Date: Tue, 14 Nov 2023 14:49:37 +0800
Subject: [PATCH 09/74] =?UTF-8?q?bug:=20=E4=BC=98=E5=8C=96DevX=E7=BC=93?=
=?UTF-8?q?=E5=AD=98=E5=88=B7=E6=96=B0=E6=93=8D=E4=BD=9C=20#1430=20(#1434)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../fs/server/utils/DevxWorkspaceUtils.kt | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt
index 8523a3ba0a..3e91b26765 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/utils/DevxWorkspaceUtils.kt
@@ -27,9 +27,6 @@
package com.tencent.bkrepo.fs.server.utils
-import com.google.common.cache.CacheBuilder
-import com.google.common.cache.CacheLoader
-import com.google.common.cache.LoadingCache
import com.tencent.bkrepo.common.api.util.toJsonString
import com.tencent.bkrepo.common.security.interceptor.devx.ApiAuth
import com.tencent.bkrepo.common.security.interceptor.devx.DevXProperties
@@ -49,6 +46,7 @@ import reactor.netty.http.client.PrematureCloseException
import reactor.netty.resources.ConnectionProvider
import reactor.util.retry.RetryBackoffSpec
import java.time.Duration
+import java.util.concurrent.ConcurrentHashMap
class DevxWorkspaceUtils(
devXProperties: DevXProperties
@@ -67,13 +65,15 @@ class DevxWorkspaceUtils(
val connector = ReactorClientHttpConnector(client)
WebClient.builder().clientConnector(connector).build()
}
- private val projectIpsCache: LoadingCache>> by lazy { CacheBuilder.newBuilder()
- .maximumSize(devXProperties.cacheSize)
- .expireAfterWrite(devXProperties.cacheExpireTime)
- .build(CacheLoader.from { key -> listIpFromProject(key) }) }
+
+ private val projectIpsCache: ConcurrentHashMap>> by lazy {
+ ConcurrentHashMap(devXProperties.cacheSize.toInt())
+ }
fun getIpList(projectId: String): Mono> {
- return projectIpsCache.get(projectId)
+ return projectIpsCache[projectId] ?: synchronized(DevxWorkspaceUtils::class) {
+ projectIpsCache.getOrPut(projectId) { listIpFromProject(projectId) }
+ }
}
suspend fun getWorkspace(): Mono {
@@ -120,10 +120,11 @@ class DevxWorkspaceUtils(
logger.warn("request ips of project[$projectId] failed, will retry: $retry")
retry
}
- )
+ ).cache(devXProperties.cacheExpireTime)
}
private suspend fun parseResponse(response: ClientResponse, projectId: String): Set {
+ logger.info("Parse project[$projectId] ips.")
return if (response.statusCode() != HttpStatus.OK) {
val errorMsg = response.awaitBody()
logger.error("${response.statusCode()} $errorMsg")
From df34a6ffea48835002e8715e3ee5073212396a01 Mon Sep 17 00:00:00 2001
From: kunlongli <16629885+cnlkl@users.noreply.github.com>
Date: Wed, 15 Nov 2023 10:28:48 +0800
Subject: [PATCH 10/74] =?UTF-8?q?bug:=20=E4=BF=AE=E5=A4=8Daccept=E4=B8=BAn?=
=?UTF-8?q?ull=E5=AF=BC=E8=87=B4=E7=9A=84=E7=A9=BA=E6=8C=87=E9=92=88?=
=?UTF-8?q?=E9=94=99=E8=AF=AF=20#1423=20(#1439)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../bkrepo/common/api/constant/MediaTypes.kt | 1 +
.../exception/AbstractExceptionHandler.kt | 20 ++++++++++++-------
2 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/MediaTypes.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/MediaTypes.kt
index 9163c5e3ae..fb34984b8b 100644
--- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/MediaTypes.kt
+++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/MediaTypes.kt
@@ -37,6 +37,7 @@ object MediaTypes {
const val APPLICATION_XML = "application/xml;charset=utf-8"
const val APPLICATION_JSON_WITHOUT_CHARSET = "application/json"
const val APPLICATION_XML_WITHOUT_CHARSET = "application/xml"
+ const val TEXT_XML = "text/xml"
const val TEXT_HTML = "text/html; charset=utf-8"
const val TEXT_PLAIN = "text/plain"
const val APPLICATION_YAML = "application/x-yaml"
diff --git a/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/AbstractExceptionHandler.kt b/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/AbstractExceptionHandler.kt
index eb342bc0d6..7c313f4492 100644
--- a/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/AbstractExceptionHandler.kt
+++ b/src/backend/common/common-service/src/main/kotlin/com/tencent/bkrepo/common/service/exception/AbstractExceptionHandler.kt
@@ -70,17 +70,23 @@ open class AbstractExceptionHandler {
return ResponseBuilder.fail(code, errorMessage)
}
- protected fun updateContentType(contentType: String? = null) {
+ protected open fun updateContentType(contentType: String? = null) {
if (contentType != null) {
HttpContextHolder.getResponse().contentType = contentType
return
}
- val accept = HttpContextHolder.getRequest().getHeader(HttpHeaders.ACCEPT)
- if (HttpContextHolder.getResponse().contentType == null &&
- !accept.startsWith(MediaTypes.APPLICATION_JSON_WITHOUT_CHARSET, true) &&
- !accept.startsWith(MediaTypes.APPLICATION_XML_WITHOUT_CHARSET, true)
- ) {
- HttpContextHolder.getResponse().contentType = MediaTypes.APPLICATION_JSON
+ val acceptHeader = HttpContextHolder.getRequest().getHeader(HttpHeaders.ACCEPT)
+ if (acceptHeader.isNullOrEmpty() || !HttpContextHolder.getResponse().contentType.isNullOrEmpty()) {
+ return
+ }
+
+ val acceptXml = acceptHeader.contains(MediaTypes.TEXT_XML, true) ||
+ acceptHeader.contains(MediaTypes.APPLICATION_XML_WITHOUT_CHARSET, true)
+
+ if (acceptXml) {
+ HttpContextHolder.getResponse().contentType = MediaTypes.APPLICATION_XML_WITHOUT_CHARSET
+ } else {
+ HttpContextHolder.getResponse().contentType = MediaTypes.APPLICATION_JSON_WITHOUT_CHARSET
}
}
}
From 901a1e17969153149b0a822bff6eee4ce39c5f47 Mon Sep 17 00:00:00 2001
From: lannoy0523 <46735290+lannoy0523@users.noreply.github.com>
Date: Wed, 15 Nov 2023 11:09:06 +0800
Subject: [PATCH 11/74] =?UTF-8?q?feat:=E7=AE=A1=E7=90=86=E5=90=8E=E5=8F=B0?=
=?UTF-8?q?=E6=94=AF=E6=8C=81=E4=BF=AE=E6=94=B9=E5=AD=98=E5=82=A8=E5=AE=9E?=
=?UTF-8?q?=E4=BE=8B=E7=9A=84=E4=B8=8B=E8=BD=BDqps=E9=85=8D=E7=BD=AE=20#14?=
=?UTF-8?q?17=20(#1419)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../components/CreateOrUpdateCredentialDialog.vue | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/src/frontend/devops-op/src/views/storage/components/CreateOrUpdateCredentialDialog.vue b/src/frontend/devops-op/src/views/storage/components/CreateOrUpdateCredentialDialog.vue
index 14f9e6b451..4ba2f3ce27 100644
--- a/src/frontend/devops-op/src/views/storage/components/CreateOrUpdateCredentialDialog.vue
+++ b/src/frontend/devops-op/src/views/storage/components/CreateOrUpdateCredentialDialog.vue
@@ -165,6 +165,19 @@
/>
+
+
+
+
@@ -376,6 +389,7 @@ export default {
credential.download.taskInterval = 10
credential.download.minimumPartSize = 10
credential.download.maxDownloadParts = 10 * 1000
+ credential.download.qps = 10
}
credential.secretKey = ''
credential.bucket = ''
From 698b69ebd2f2c9bd1412c5739a314eeabdb5398c Mon Sep 17 00:00:00 2001
From: gujunling <34835140+gujunling@users.noreply.github.com>
Date: Wed, 15 Nov 2023 15:21:37 +0800
Subject: [PATCH 12/74] =?UTF-8?q?fix:=20=E5=88=9B=E5=BB=BA=E5=88=86?=
=?UTF-8?q?=E5=8F=91=E8=AE=A1=E5=88=92=E6=97=B6cron=E8=A1=A8=E8=BE=BE?=
=?UTF-8?q?=E5=BC=8F=E7=9A=84=E8=8B=B1=E6=96=87=E7=BF=BB=E8=AF=91=E6=9C=89?=
=?UTF-8?q?=E8=AF=AF=20#1403=20(#1404)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* fix: 创建分发计划时cron表达式的英文翻译有误 #1403
* fix: 补充corn表达式的中文翻译更改 #1403
---
.../src/components/Cron/index.vue | 22 +++++++++----------
src/frontend/devops-repository/src/main.js | 3 +++
src/frontend/locale/repository/en-US.json | 11 ++++++++--
src/frontend/locale/repository/zh-CN.json | 14 ++++++------
4 files changed, 30 insertions(+), 20 deletions(-)
diff --git a/src/frontend/devops-repository/src/components/Cron/index.vue b/src/frontend/devops-repository/src/components/Cron/index.vue
index a057c5153d..dd3850345a 100644
--- a/src/frontend/devops-repository/src/components/Cron/index.vue
+++ b/src/frontend/devops-repository/src/components/Cron/index.vue
@@ -4,7 +4,7 @@
-
+
{{ $t('manualInput')}}{{ $t('autoEdit') }}
@@ -28,14 +28,14 @@
{{ $t(`cron.${type}`, [$t(`cron.${tab === 'week' ? 'day' : tab}`)]) }}{{ $t('cron.from') }}
- {{ $t('cron.to') }}
- {{ $t('intervalMsg') }}
- {{ $t('circleMsg1') }}
-
Date: Wed, 15 Nov 2023 17:18:06 +0800
Subject: [PATCH 13/74] =?UTF-8?q?bug:=20=E4=BF=AE=E5=A4=8D=E5=88=9B?=
=?UTF-8?q?=E5=BB=BAk8s=20Secret=E6=97=B6=E7=BC=BA=E5=B0=91kind=E5=8F=82?=
=?UTF-8?q?=E6=95=B0=20#1444=20(#1445)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../main/kotlin/com/tencent/bkrepo/analyst/utils/K8SHelper.kt | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/K8SHelper.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/K8SHelper.kt
index 250daab9e4..5bc40eb9cf 100644
--- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/K8SHelper.kt
+++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/K8SHelper.kt
@@ -64,6 +64,8 @@ class K8SHelper(k8sProp: KubernetesExecutionClusterProperties) {
}
""".trimIndent().jsonCompress()
val pullSecret = V1Secret {
+ apiVersion = "v1"
+ kind = "Secret"
metadata {
name = secretName
namespace = this@K8SHelper.namespace
From 957906eaf68c10580e37193b0ffad86c760d5d37 Mon Sep 17 00:00:00 2001
From: owen
Date: Thu, 16 Nov 2023 20:10:41 +0800
Subject: [PATCH 14/74] =?UTF-8?q?bug:=20devx=20login=20=E9=BB=98=E8=AE=A4?=
=?UTF-8?q?=E5=88=9B=E5=BB=BA=E7=94=A8=E6=88=B7=E5=88=B0=E5=88=B6=E5=93=81?=
=?UTF-8?q?=E5=BA=93=20#1452=20(#1453)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../com/tencent/bkrepo/fs/server/api/RAuthClient.kt | 6 ++++++
.../com/tencent/bkrepo/fs/server/handler/LoginHandler.kt | 9 ++++++++-
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RAuthClient.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RAuthClient.kt
index ae95ca0f19..adc62ee315 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RAuthClient.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RAuthClient.kt
@@ -28,6 +28,7 @@
package com.tencent.bkrepo.fs.server.api
import com.tencent.bkrepo.auth.pojo.permission.CheckPermissionRequest
+import com.tencent.bkrepo.auth.pojo.user.CreateUserRequest
import com.tencent.bkrepo.auth.pojo.user.User
import com.tencent.bkrepo.common.api.constant.AUTH_SERVICE_NAME
import com.tencent.bkrepo.common.api.pojo.Response
@@ -61,4 +62,9 @@ interface RAuthClient {
fun detail(
@PathVariable uid: String
): Mono>
+
+ @PostMapping("/user/create")
+ fun create(
+ @RequestBody request: CreateUserRequest
+ ): Mono>
}
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt
index 2bcce18bc0..73245836ae 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/LoginHandler.kt
@@ -28,6 +28,7 @@
package com.tencent.bkrepo.fs.server.handler
import com.tencent.bkrepo.auth.pojo.enums.PermissionAction
+import com.tencent.bkrepo.auth.pojo.user.CreateUserRequest
import com.tencent.bkrepo.common.api.constant.BASIC_AUTH_PREFIX
import com.tencent.bkrepo.common.api.constant.HttpHeaders
import com.tencent.bkrepo.common.api.util.BasicAuthUtils
@@ -81,11 +82,17 @@ class LoginHandler(
suspend fun devxLogin(request: ServerRequest): ServerResponse {
val workspace = DevxWorkspaceUtils.getWorkspace().awaitSingleOrNull() ?: throw AuthenticationException()
val repoName = request.pathVariable(REPO_NAME)
+ createUser(workspace.owner)
val token = createToken(workspace.projectId, repoName, workspace.owner)
val response = DevxLoginResponse(workspace.projectId, token)
return ReactiveResponseBuilder.success(response)
}
+ private suspend fun createUser(userName: String) {
+ val request = CreateUserRequest(userId = userName, name = userName)
+ rAuthClient.create(request).awaitSingle()
+ }
+
private suspend fun createToken(projectId: String, repoName: String, username: String): String {
val claims = mutableMapOf(JWT_CLAIMS_REPOSITORY to "$projectId/$repoName")
val writePermit = permissionService.checkPermission(projectId, repoName, PermissionAction.WRITE, username)
@@ -94,7 +101,7 @@ class LoginHandler(
} else {
val repoDetail = ReactiveArtifactContextHolder.getRepoDetail()
val readPermit = repoDetail.public ||
- permissionService.checkPermission(projectId, repoName, PermissionAction.READ, username)
+ permissionService.checkPermission(projectId, repoName, PermissionAction.READ, username)
if (readPermit) {
claims[JWT_CLAIMS_PERMIT] = PermissionAction.READ.name
}
From 2d180c019fb621c96e5845e143a1d595186d8d18 Mon Sep 17 00:00:00 2001
From: zacYL <100330102+zacYL@users.noreply.github.com>
Date: Thu, 16 Nov 2023 20:37:52 +0800
Subject: [PATCH 15/74] =?UTF-8?q?feat:=20=E4=BB=93=E5=BA=93=E7=BB=9F?=
=?UTF-8?q?=E8=AE=A1=E5=AD=98=E5=82=A8=E9=80=BB=E8=BE=91=E8=B0=83=E6=95=B4?=
=?UTF-8?q?=20#1435=20(#1441)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../job/batch/ProjectRepoMetricsStatJob.kt | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectRepoMetricsStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectRepoMetricsStatJob.kt
index 32ceab91e9..aa7946e446 100644
--- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectRepoMetricsStatJob.kt
+++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectRepoMetricsStatJob.kt
@@ -43,6 +43,7 @@ import com.tencent.bkrepo.job.batch.utils.TimeUtils
import com.tencent.bkrepo.job.config.properties.ProjectRepoMetricsStatJobProperties
import org.slf4j.LoggerFactory
import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.data.mongodb.core.BulkOperations
import org.springframework.data.mongodb.core.MongoTemplate
import org.springframework.data.mongodb.core.query.Criteria
import org.springframework.data.mongodb.core.query.Query
@@ -112,10 +113,13 @@ class ProjectRepoMetricsStatJob(
super.onRunCollectionFinished(collectionName, context)
require(context is ProjectRepoMetricsStatJobContext)
logger.info("start to insert project's metrics ")
+ val projectMetrics = ArrayList(context.metrics.size)
for (entry in context.metrics) {
- storeMetrics(context.statDate, entry.value)
+ projectMetrics.add(entry.value.toDO(context.statDate))
}
+ storeMetrics(context.statDate, projectMetrics)
context.metrics.clear()
+ projectMetrics.clear()
logger.info("stat project metrics done")
}
@@ -127,14 +131,14 @@ class ProjectRepoMetricsStatJob(
private fun storeMetrics(
statDate: LocalDateTime,
- projectMetric: ProjectRepoMetricsStatJobContext.ProjectMetrics
+ projectMetrics: List
) {
- val projectId = projectMetric.projectId
// insert project repo metrics
- val criteria = Criteria.where(PROJECT).isEqualTo(projectId).and(CREATED_DATE).isEqualTo(statDate)
+ val criteria = Criteria.where(CREATED_DATE).isEqualTo(statDate)
mongoTemplate.remove(Query(criteria), COLLECTION_NAME_PROJECT_METRICS)
- logger.info("stat project: [${projectId}], size: [${projectMetric.capSize.toLong()}]")
- mongoTemplate.insert(projectMetric.toDO(statDate), COLLECTION_NAME_PROJECT_METRICS)
+ mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, COLLECTION_NAME_PROJECT_METRICS)
+ .insert(projectMetrics)
+ .execute()
}
data class Repository(
From 311576c49af73d835cdddbb43ab6dd3162b1eda4 Mon Sep 17 00:00:00 2001
From: zacYL <100330102+zacYL@users.noreply.github.com>
Date: Thu, 16 Nov 2023 20:38:04 +0800
Subject: [PATCH 16/74] =?UTF-8?q?feat:=20=E7=A9=BA=E7=9B=AE=E5=BD=95?=
=?UTF-8?q?=E6=B8=85=E7=90=86=E4=BB=BB=E5=8A=A1=E9=80=BB=E8=BE=91=E8=B0=83?=
=?UTF-8?q?=E6=95=B4=20#1440=20(#1446)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: 空目录清理任务逻辑调整 #1440
* feat: 代码调整 #1440
* feat: 代码调整 #1440
* feat: 默认generic仓库避免查询 #1440
---
.../batch/{node => }/EmptyFolderCleanupJob.kt | 173 +++++++++++-------
...ext.kt => EmptyFolderCleanupJobContext.kt} | 11 +-
.../node/NodeStatCompositeMongoDbBatchJob.kt | 4 +-
.../EmptyFolderCleanupJobProperties.kt | 39 ++++
4 files changed, 155 insertions(+), 72 deletions(-)
rename src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/{node => }/EmptyFolderCleanupJob.kt (52%)
rename src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/{EmptyFolderChildContext.kt => EmptyFolderCleanupJobContext.kt} (90%)
create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/EmptyFolderCleanupJobProperties.kt
diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/EmptyFolderCleanupJob.kt
similarity index 52%
rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt
rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/EmptyFolderCleanupJob.kt
index 8ca8aa3ac4..59436a0ed7 100644
--- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt
+++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/EmptyFolderCleanupJob.kt
@@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
-package com.tencent.bkrepo.job.batch.node
+package com.tencent.bkrepo.job.batch
import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.artifact.constant.CUSTOM
@@ -33,105 +33,134 @@ import com.tencent.bkrepo.common.artifact.constant.LOG
import com.tencent.bkrepo.common.artifact.constant.PIPELINE
import com.tencent.bkrepo.common.artifact.constant.REPORT
import com.tencent.bkrepo.common.artifact.path.PathUtils
-import com.tencent.bkrepo.common.mongo.constant.ID
-import com.tencent.bkrepo.common.service.log.LoggerHolder
+import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
import com.tencent.bkrepo.job.DELETED_DATE
import com.tencent.bkrepo.job.FOLDER
import com.tencent.bkrepo.job.FULLPATH
-import com.tencent.bkrepo.job.LAST_MODIFIED_DATE
import com.tencent.bkrepo.job.PROJECT
import com.tencent.bkrepo.job.REPO
-import com.tencent.bkrepo.job.batch.base.ChildJobContext
-import com.tencent.bkrepo.job.batch.base.ChildMongoDbBatchJob
+import com.tencent.bkrepo.job.SHARDING_COUNT
+import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob
import com.tencent.bkrepo.job.batch.base.JobContext
-import com.tencent.bkrepo.job.batch.context.EmptyFolderChildContext
-import com.tencent.bkrepo.job.batch.utils.FolderUtils.buildCacheKey
-import com.tencent.bkrepo.job.batch.utils.FolderUtils.extractFolderInfoFromCacheKey
-import com.tencent.bkrepo.job.config.properties.CompositeJobProperties
-import org.bson.types.ObjectId
+import com.tencent.bkrepo.job.batch.context.EmptyFolderCleanupJobContext
+import com.tencent.bkrepo.job.batch.utils.FolderUtils
+import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils
+import com.tencent.bkrepo.job.config.properties.EmptyFolderCleanupJobProperties
+import com.tencent.bkrepo.repository.api.NodeClient
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest
+import org.slf4j.LoggerFactory
+import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.data.mongodb.core.MongoTemplate
import org.springframework.data.mongodb.core.query.Criteria
import org.springframework.data.mongodb.core.query.Query
-import org.springframework.data.mongodb.core.query.Update
import org.springframework.data.mongodb.core.query.isEqualTo
-import java.time.LocalDateTime
+import org.springframework.stereotype.Component
+import java.time.Duration
+
/**
* 空目录清理job
*/
+@Component
+@EnableConfigurationProperties(EmptyFolderCleanupJobProperties::class)
class EmptyFolderCleanupJob(
- val properties: CompositeJobProperties,
+ properties: EmptyFolderCleanupJobProperties,
private val mongoTemplate: MongoTemplate,
-) : ChildMongoDbBatchJob(properties) {
- override fun onParentJobStart(context: ChildJobContext) {
- logger.info("start to run emptyFolderCleanupJob")
+ private val nodeClient: NodeClient
+): DefaultContextMongoDbJob(properties) {
+
+ override fun collectionNames(): List {
+ return (0 until SHARDING_COUNT).map { "$COLLECTION_NAME_PREFIX$it" }.toList()
+ }
+
+ override fun buildQuery(): Query = Query()
+
+ override fun mapToEntity(row: Map): Node {
+ return Node(row)
+ }
+
+ override fun entityClass(): Class {
+ return Node::class.java
}
- override fun run(row: NodeStatCompositeMongoDbBatchJob.Node, collectionName: String, context: JobContext) {
- require(context is EmptyFolderChildContext)
+ override fun run(row: Node, collectionName: String, context: JobContext) {
+ require(context is EmptyFolderCleanupJobContext)
if (row.deleted != null) return
- if (row.repoName !in TARGET_REPO_LIST) return
+ // 暂时只清理generic类型仓库下的空目录
+ if (row.repoName !in TARGET_REPO_LIST && RepositoryCommonUtils.getRepositoryDetail(
+ row.projectId, row.repoName
+ ).type != RepositoryType.GENERIC) return
if (row.folder) {
- val folderKey = buildCacheKey(
+ val folderKey = FolderUtils.buildCacheKey(
collectionName = collectionName, projectId = row.projectId,
repoName = row.repoName, fullPath = row.fullPath
)
- val folderMetric = context.folders.getOrPut(folderKey) {
- EmptyFolderChildContext.FolderMetricsInfo(id = row.id)
+ context.folders.getOrPut(folderKey) {
+ EmptyFolderCleanupJobContext.FolderMetricsInfo()
}
- folderMetric.id = row.id
} else {
val ancestorFolder = PathUtils.resolveAncestor(row.fullPath)
for (folder in ancestorFolder) {
if (folder == PathUtils.ROOT) continue
val tempFullPath = PathUtils.toFullPath(folder)
- val folderKey = buildCacheKey(
+ val folderKey = FolderUtils.buildCacheKey(
collectionName = collectionName, projectId = row.projectId,
repoName = row.repoName, fullPath = tempFullPath
)
- val folderMetric = context.folders.getOrPut(folderKey) { EmptyFolderChildContext.FolderMetricsInfo() }
+ val folderMetric = context.folders.getOrPut(folderKey) {
+ EmptyFolderCleanupJobContext.FolderMetricsInfo()
+ }
folderMetric.nodeNum.increment()
}
}
}
- override fun onParentJobFinished(context: ChildJobContext) {
- require(context is EmptyFolderChildContext)
- logger.info("emptyFolderCleanupJob finished, deleted empty folder num: ${context.totalDeletedNum}")
+ override fun getLockAtMostFor(): Duration {
+ return Duration.ofDays(7)
}
- override fun createChildJobContext(parentJobContext: JobContext): ChildJobContext {
- return EmptyFolderChildContext(parentJobContext)
+
+ override fun createJobContext(): EmptyFolderCleanupJobContext {
+ return EmptyFolderCleanupJobContext()
}
+
+
override fun onRunCollectionFinished(collectionName: String, context: JobContext) {
- require(context is EmptyFolderChildContext)
+ require(context is EmptyFolderCleanupJobContext)
super.onRunCollectionFinished(collectionName, context)
- filterEmptyFolders(collectionName, context)
+ deleteEmptyFolders(collectionName, context)
}
- private fun filterEmptyFolders(
+ private fun deleteEmptyFolders(
collectionName: String,
- context: EmptyFolderChildContext
+ context: EmptyFolderCleanupJobContext
) {
logger.info("will filter empty folder in table $collectionName")
if (context.folders.isEmpty()) return
- val prefix = buildCacheKey(collectionName = collectionName, projectId = StringPool.EMPTY)
+ val prefix = FolderUtils.buildCacheKey(collectionName = collectionName, projectId = StringPool.EMPTY)
for(entry in context.folders) {
if (!entry.key.startsWith(prefix) || entry.value.nodeNum.toLong() > 0) continue
- extractFolderInfoFromCacheKey(entry.key)?.let {
- if (emptyFolderDoubleCheck(
- projectId = it.projectId,
- repoName = it.repoName,
- path = it.fullPath,
- collectionName = collectionName
+ val folderInfo = FolderUtils.extractFolderInfoFromCacheKey(entry.key) ?: continue
+ if (emptyFolderDoubleCheck(
+ projectId = folderInfo.projectId,
+ repoName = folderInfo.repoName,
+ path = folderInfo.fullPath,
+ collectionName = collectionName
)) {
- logger.info("will delete empty folder ${it.fullPath} in repo ${it.projectId}|${it.repoName}")
+ logger.info("will delete empty folder ${folderInfo.fullPath}" +
+ " in repo ${folderInfo.projectId}|${folderInfo.repoName}")
+ val deletedResult = doEmptyFolderDelete(
+ projectId = folderInfo.projectId,
+ repoName = folderInfo.repoName,
+ fullPath = folderInfo.fullPath
+ )
+ if (deletedResult) {
context.totalDeletedNum.increment()
- deleteEmptyFolders(entry.value.id, collectionName)
}
}
}
@@ -163,20 +192,19 @@ class EmptyFolderCleanupJob(
/**
* 删除空目录
*/
- private fun deleteEmptyFolders(
- objectId: String?,
- collectionName: String
- ) {
- if (objectId.isNullOrEmpty()) return
- val query = Query(
- Criteria.where(ID).isEqualTo(ObjectId(objectId))
- .and(FOLDER).isEqualTo(true)
- )
- val deleteTime = LocalDateTime.now()
- val update = Update()
- .set(LAST_MODIFIED_DATE, deleteTime)
- .set(DELETED_DATE, deleteTime)
- mongoTemplate.updateFirst(query, update, collectionName)
+ private fun doEmptyFolderDelete(
+ projectId: String,
+ repoName: String,
+ fullPath: String,
+ ): Boolean {
+ return try {
+ // 删除空目录改为调用接口删除，避免监听删除事件的场景无法触发
+ nodeClient.deleteNode(NodeDeleteRequest(projectId, repoName, fullPath, SYSTEM_USER))
+ true
+ } catch (e: Exception) {
+ logger.info("Failed to delete empty folder $fullPath in repo $projectId|$repoName, error is $e")
+ false
+ }
}
/**
@@ -184,9 +212,9 @@ class EmptyFolderCleanupJob(
*/
private fun clearCollectionContextCache(
collectionName: String,
- context: EmptyFolderChildContext
+ context: EmptyFolderCleanupJobContext
) {
- val prefix = buildCacheKey(collectionName = collectionName, projectId = StringPool.EMPTY)
+ val prefix = FolderUtils.buildCacheKey(collectionName = collectionName, projectId = StringPool.EMPTY)
for(entry in context.folders) {
if (entry.key.contains(prefix))
context.folders.remove(entry.key)
@@ -194,11 +222,28 @@ class EmptyFolderCleanupJob(
}
+ data class Node(
+ val projectId: String,
+ val repoName: String,
+ val folder: Boolean,
+ val fullPath: String,
+ val deleted: String? = null
+ ) {
+ constructor(map: Map) : this(
+ map[Node::projectId.name].toString(),
+ map[Node::repoName.name].toString(),
+ map[Node::folder.name] as Boolean,
+ map[Node::fullPath.name].toString(),
+ map[Node::deleted.name]?.toString(),
+ )
+ }
+
+
companion object {
- private val logger = LoggerHolder.jobLogger
+ private val logger = LoggerFactory.getLogger(EmptyFolderCleanupJob::class.java)
const val FULL_PATH_IDX = "projectId_repoName_fullPath_idx"
- private val TARGET_REPO_LIST = listOf(REPORT, LOG, PIPELINE, CUSTOM)
-
+ private const val COLLECTION_NAME_PREFIX = "node_"
+ private val TARGET_REPO_LIST = listOf(REPORT, LOG, PIPELINE, CUSTOM, "remote-mirrors")
}
-}
+}
\ No newline at end of file
diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderCleanupJobContext.kt
similarity index 90%
rename from src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt
rename to src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderCleanupJobContext.kt
index c5a6968ea8..5dc31aad8d 100644
--- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt
+++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderCleanupJobContext.kt
@@ -27,21 +27,22 @@
package com.tencent.bkrepo.job.batch.context
-import com.tencent.bkrepo.job.batch.base.ChildJobContext
import com.tencent.bkrepo.job.batch.base.JobContext
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.LongAdder
-class EmptyFolderChildContext(
- parentContent: JobContext,
+data class EmptyFolderCleanupJobContext(
// 用于内存缓存下存储目录统计信息
var folders: ConcurrentHashMap = ConcurrentHashMap(),
// 总共删除的路径个数
var totalDeletedNum: LongAdder = LongAdder()
-): ChildJobContext(parentContent) {
+) : JobContext() {
data class FolderMetricsInfo(
- var id: String? = null,
var nodeNum: LongAdder = LongAdder()
)
+
+ override fun toString(): String {
+ return "Success[$success], failed[$failed], total[$total], deletedFolderNum[${totalDeletedNum.toLong()}]"
+ }
}
diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt
index 2b85abe511..58c9f5deb3 100644
--- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt
+++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt
@@ -32,8 +32,7 @@ class NodeStatCompositeMongoDbBatchJob(
override fun createChildJobs(): List> {
return listOf(
- FolderStatChildJob(properties, mongoTemplate, redisTemplate),
- EmptyFolderCleanupJob(properties, mongoTemplate)
+ FolderStatChildJob(properties, mongoTemplate, redisTemplate)
)
}
@@ -83,6 +82,5 @@ class NodeStatCompositeMongoDbBatchJob(
projectId = map[Node::projectId.name] as String
repoName = map[Node::repoName.name] as String
}
-
}
}
diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/EmptyFolderCleanupJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/EmptyFolderCleanupJobProperties.kt
new file mode 100644
index 0000000000..074c787b62
--- /dev/null
+++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/EmptyFolderCleanupJobProperties.kt
@@ -0,0 +1,39 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.config.properties
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+/**
+ * 空目录清理任务配置项
+ */
+@ConfigurationProperties("job.empty-folder-cleanup")
+data class EmptyFolderCleanupJobProperties(
+ override var enabled: Boolean = true,
+ override var cron: String = "0 0 18 * * ?"
+) : MongodbJobProperties()
\ No newline at end of file
From c34518c40140adb29a0e7eb99a7908c5b5a60f49 Mon Sep 17 00:00:00 2001
From: yaoxuwan
Date: Thu, 16 Nov 2023 20:40:36 +0800
Subject: [PATCH 17/74] =?UTF-8?q?feat:=20stat=E6=94=B9=E4=B8=BA=E5=8F=96no?=
=?UTF-8?q?de=E5=BF=AB=E7=85=A7=E6=95=B0=E6=8D=AE=20#1282=20(#1431)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: stat改为取node快照数据 #1282
* feat: stat改为取node快照数据 #1282
---
.../fs/server/handler/NodeOperationsHandler.kt | 5 +++--
.../service/repo/impl/RepositoryServiceImpl.kt | 16 ++--------------
.../CommitEdgeCenterRepositoryServiceImpl.kt | 7 ++-----
.../repo/impl/edge/EdgeRepositoryServiceImpl.kt | 7 ++-----
4 files changed, 9 insertions(+), 26 deletions(-)
diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt
index 128e0e4bc5..f8e4b92166 100644
--- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt
+++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt
@@ -50,6 +50,7 @@ import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder
import com.tencent.bkrepo.fs.server.utils.ReactiveSecurityUtils
import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel
import com.tencent.bkrepo.repository.pojo.metadata.MetadataSaveRequest
+import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo
import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest
import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest
@@ -156,10 +157,10 @@ class NodeOperationsHandler(
if (res == null) {
val cap = ReactiveArtifactContextHolder.getRepoDetail().quota
val nodeStat = try {
- rRepositoryClient.computeSize(projectId, repoName, fullPath, true).awaitSingle().data
+ rRepositoryClient.statRepo(projectId, repoName).awaitSingle().data
} catch (e: ReadTimeoutException) {
logger.warn("get repo[$projectId/$repoName] stat timeout")
- rRepositoryClient.statRepo(projectId, repoName).awaitSingle().data
+ NodeSizeInfo(0,0, UNKNOWN)
}
res = StatResponse(
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
index 736c78e63f..7cff5657db 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt
@@ -28,7 +28,6 @@
package com.tencent.bkrepo.repository.service.repo.impl
import com.tencent.bkrepo.auth.api.ServicePermissionClient
-import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.api.exception.ErrorCodeException
import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.api.pojo.Page
@@ -58,11 +57,9 @@ import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publi
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier
import com.tencent.bkrepo.repository.config.RepositoryProperties
-import com.tencent.bkrepo.repository.dao.NodeDao
import com.tencent.bkrepo.repository.dao.RepositoryDao
import com.tencent.bkrepo.repository.dao.repository.ProjectMetricsRepository
import com.tencent.bkrepo.repository.model.TRepository
-import com.tencent.bkrepo.repository.pojo.node.NodeListOption
import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo
import com.tencent.bkrepo.repository.pojo.project.RepoRangeQueryRequest
import com.tencent.bkrepo.repository.pojo.proxy.ProxyChannelCreateRequest
@@ -80,7 +77,6 @@ import com.tencent.bkrepo.repository.service.repo.ProjectService
import com.tencent.bkrepo.repository.service.repo.ProxyChannelService
import com.tencent.bkrepo.repository.service.repo.RepositoryService
import com.tencent.bkrepo.repository.service.repo.StorageCredentialService
-import com.tencent.bkrepo.repository.util.NodeQueryHelper
import com.tencent.bkrepo.repository.util.RepoEventFactory.buildCreatedEvent
import com.tencent.bkrepo.repository.util.RepoEventFactory.buildDeletedEvent
import com.tencent.bkrepo.repository.util.RepoEventFactory.buildUpdatedEvent
@@ -116,8 +112,7 @@ class RepositoryServiceImpl(
private val repositoryProperties: RepositoryProperties,
private val messageSupplier: MessageSupplier,
private val servicePermissionClient: ServicePermissionClient,
- private val projectMetricsRepository: ProjectMetricsRepository,
- private val nodeDao: NodeDao
+ private val projectMetricsRepository: ProjectMetricsRepository
) : RepositoryService {
init {
@@ -377,17 +372,10 @@ class RepositoryServiceImpl(
}
override fun statRepo(projectId: String, repoName: String): NodeSizeInfo {
- val query = Query(NodeQueryHelper.nodeListCriteria(
- projectId = projectId,
- repoName = repoName,
- path = StringPool.ROOT,
- option = NodeListOption(includeFolder = true, deep = true)
- ))
- val count = nodeDao.count(query)
val projectMetrics = projectMetricsRepository.findFirstByProjectIdOrderByCreatedDateDesc(projectId)
val repoMetrics = projectMetrics?.repoMetrics?.firstOrNull { it.repoName == repoName }
return NodeSizeInfo(
- subNodeCount = count,
+ subNodeCount = repoMetrics?.num ?: 0,
subNodeWithoutFolderCount = repoMetrics?.num ?: 0,
size = repoMetrics?.size ?: 0
)
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/center/CommitEdgeCenterRepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/center/CommitEdgeCenterRepositoryServiceImpl.kt
index e423817588..33da95ed43 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/center/CommitEdgeCenterRepositoryServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/center/CommitEdgeCenterRepositoryServiceImpl.kt
@@ -45,7 +45,6 @@ import com.tencent.bkrepo.common.service.cluster.CommitEdgeCenterCondition
import com.tencent.bkrepo.common.service.util.SpringContextUtils
import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier
import com.tencent.bkrepo.repository.config.RepositoryProperties
-import com.tencent.bkrepo.repository.dao.NodeDao
import com.tencent.bkrepo.repository.dao.RepositoryDao
import com.tencent.bkrepo.repository.dao.repository.ProjectMetricsRepository
import com.tencent.bkrepo.repository.model.TRepository
@@ -75,8 +74,7 @@ class CommitEdgeCenterRepositoryServiceImpl(
messageSupplier: MessageSupplier,
servicePermissionClient: ServicePermissionClient,
private val clusterProperties: ClusterProperties,
- projectMetricsRepository: ProjectMetricsRepository,
- nodeDao: NodeDao
+ projectMetricsRepository: ProjectMetricsRepository
) : RepositoryServiceImpl(
repositoryDao,
nodeService,
@@ -86,8 +84,7 @@ class CommitEdgeCenterRepositoryServiceImpl(
repositoryProperties,
messageSupplier,
servicePermissionClient,
- projectMetricsRepository,
- nodeDao
+ projectMetricsRepository
) {
override fun determineStorageKey(request: RepoCreateRequest): String? {
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/edge/EdgeRepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/edge/EdgeRepositoryServiceImpl.kt
index de898d98f3..aeb678735d 100644
--- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/edge/EdgeRepositoryServiceImpl.kt
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/edge/EdgeRepositoryServiceImpl.kt
@@ -36,7 +36,6 @@ import com.tencent.bkrepo.common.service.feign.FeignClientFactory
import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier
import com.tencent.bkrepo.repository.api.cluster.ClusterRepositoryClient
import com.tencent.bkrepo.repository.config.RepositoryProperties
-import com.tencent.bkrepo.repository.dao.NodeDao
import com.tencent.bkrepo.repository.dao.RepositoryDao
import com.tencent.bkrepo.repository.dao.repository.ProjectMetricsRepository
import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest
@@ -63,8 +62,7 @@ class EdgeRepositoryServiceImpl(
messageSupplier: MessageSupplier,
servicePermissionClient: ServicePermissionClient,
clusterProperties: ClusterProperties,
- projectMetricsRepository: ProjectMetricsRepository,
- nodeDao: NodeDao
+ projectMetricsRepository: ProjectMetricsRepository
) : RepositoryServiceImpl(
repositoryDao,
nodeService,
@@ -74,8 +72,7 @@ class EdgeRepositoryServiceImpl(
repositoryProperties,
messageSupplier,
servicePermissionClient,
- projectMetricsRepository,
- nodeDao
+ projectMetricsRepository
) {
private val centerRepoClient: ClusterRepositoryClient by lazy {
From b57006621bbd04bc93239048c04581471856e824 Mon Sep 17 00:00:00 2001
From: yaoxuwan
Date: Fri, 17 Nov 2023 09:44:23 +0800
Subject: [PATCH 18/74] =?UTF-8?q?fix:=20Lfs=E4=BB=93=E5=BA=93=E7=BC=93?=
=?UTF-8?q?=E5=AD=98=E6=96=87=E4=BB=B6Http=E5=93=8D=E5=BA=94=E6=9C=AA?=
=?UTF-8?q?=E5=85=B3=E9=97=AD=20#1353=20(#1424)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../tencent/bkrepo/lfs/artifact/LfsRemoteRepository.kt | 9 +++++----
.../com/tencent/bkrepo/lfs/security/RemoteAuthHandler.kt | 8 +-------
2 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/artifact/LfsRemoteRepository.kt b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/artifact/LfsRemoteRepository.kt
index caaa4fd367..987f7f8097 100644
--- a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/artifact/LfsRemoteRepository.kt
+++ b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/artifact/LfsRemoteRepository.kt
@@ -119,10 +119,11 @@ class LfsRemoteRepository : RemoteRepository() {
val actionDetail = lfsObject.actions!![DOWNLOAD_OPERATION]!!
with(actionDetail) {
val request = Request.Builder().url(href).headers(header.toHeaders()).get().build()
- val response = httpClient.newCall(request).execute()
- return if (checkResponse(response)) {
- onDownloadResponse(context, response, lfsObject.size)
- } else null
+ httpClient.newCall(request).execute().use {
+ return if (checkResponse(it)) {
+ onDownloadResponse(context, it, lfsObject.size)
+ } else null
+ }
}
}
}
diff --git a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/security/RemoteAuthHandler.kt b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/security/RemoteAuthHandler.kt
index 703fe1ff84..345e7152ab 100644
--- a/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/security/RemoteAuthHandler.kt
+++ b/src/backend/lfs/biz-lfs/src/main/kotlin/com/tencent/bkrepo/lfs/security/RemoteAuthHandler.kt
@@ -29,7 +29,6 @@ package com.tencent.bkrepo.lfs.security
import com.tencent.bkrepo.common.api.constant.BASIC_AUTH_PREFIX
import com.tencent.bkrepo.common.api.constant.HttpHeaders
-import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.api.util.BasicAuthUtils
import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
@@ -70,11 +69,6 @@ class RemoteAuthHandler : HttpAuthHandler {
override fun onAuthenticate(request: HttpServletRequest, authCredentials: HttpAuthCredentials): String {
require(authCredentials is BasicAuthCredentials)
- val passwordLength = authCredentials.password.length
- val maskPassword = authCredentials.password.replaceRange(
- IntRange(passwordLength / 2, passwordLength - 1),
- StringPool.POUND.repeat(5)
- )
- return "${authCredentials.username}:$maskPassword"
+ return authCredentials.username
}
}
From 82bd1341abf86600509dc2519d471a77e2d42ab5 Mon Sep 17 00:00:00 2001
From: yaoxuwan
Date: Fri, 17 Nov 2023 09:45:06 +0800
Subject: [PATCH 19/74] =?UTF-8?q?fix:=20fs-server=E5=9C=A8=E6=9C=AA?=
=?UTF-8?q?=E9=85=8D=E7=BD=AEinfluxdb=E6=97=B6=E6=8A=A5=E9=94=99=20#1355?=
=?UTF-8?q?=20(#1449)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../fs/boot-fs-server/src/main/resources/application.properties | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/backend/fs/boot-fs-server/src/main/resources/application.properties b/src/backend/fs/boot-fs-server/src/main/resources/application.properties
index 3de3a5d047..55f8920e96 100644
--- a/src/backend/fs/boot-fs-server/src/main/resources/application.properties
+++ b/src/backend/fs/boot-fs-server/src/main/resources/application.properties
@@ -5,3 +5,4 @@ management.endpoints.web.exposure.include=*
management.endpoints.jmx.exposure.exclude=*
management.endpoint.health.show-details=always
management.endpoint.shutdown.enabled=true
+management.metrics.export.influx.enabled=false
From e9af7d10f4156a9475c5c282c601580a22f1588c Mon Sep 17 00:00:00 2001
From: lannoy0523 <46735290+lannoy0523@users.noreply.github.com>
Date: Mon, 20 Nov 2023 14:36:31 +0800
Subject: [PATCH 20/74] =?UTF-8?q?feat:=E5=88=B6=E5=93=81=E6=90=9C=E7=B4=A2?=
=?UTF-8?q?=E9=A1=B5=E9=9D=A2=E6=90=9C=E7=B4=A2=20generic=20=E7=B1=BB?=
=?UTF-8?q?=E5=9E=8B=E5=88=B6=E5=93=81=E6=9D=A1=E4=BB=B6=E8=B0=83=E6=95=B4?=
=?UTF-8?q?=20#1370=20(#1388)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat:制品搜索页面搜索 generic 类型制品条件调整 #1370
* feat:制品搜索页面搜索 generic 类型制品条件调整 #1370
* feat:制品搜索页面搜索 generic 类型制品条件调整 #1370
---
.../src/store/actions/repoCommon.js | 8 +-
.../src/views/repoGeneric/index.vue | 89 ++++++++++++-------
.../src/views/repoSearch/index.vue | 29 +++++-
3 files changed, 85 insertions(+), 41 deletions(-)
diff --git a/src/frontend/devops-repository/src/store/actions/repoCommon.js b/src/frontend/devops-repository/src/store/actions/repoCommon.js
index 76d190ddac..9ea863cc9c 100644
--- a/src/frontend/devops-repository/src/store/actions/repoCommon.js
+++ b/src/frontend/devops-repository/src/store/actions/repoCommon.js
@@ -97,7 +97,7 @@ export default {
)
},
// 跨仓库搜索
- searchPackageList (_, { projectId, repoType, repoName, packageName, property = 'name', direction = 'ASC', current = 1, limit = 20, extRules = [] }) {
+ searchPackageList (_, { projectId, repoType, repoName, repoNames = [], packageName, property = 'name', direction = 'ASC', current = 1, limit = 20, extRules = [] }) {
const isGeneric = repoType === 'generic'
return Vue.prototype.$ajax.post(
`${prefix}/${isGeneric ? 'node/queryWithoutCount' : 'package/search'}`,
@@ -119,7 +119,7 @@ export default {
operation: 'EQ'
}]
: []),
- ...(repoType
+ ...((MODE_CONFIG === 'ci' ? !isGeneric : true) && repoType
? [{
field: 'repoType',
value: repoType.toUpperCase(),
@@ -136,8 +136,8 @@ export default {
...(MODE_CONFIG === 'ci' && isGeneric
? [{
field: 'repoName',
- value: ['report', 'log'],
- operation: 'NIN'
+ value: repoNames,
+ operation: 'IN'
}]
: [])
]),
diff --git a/src/frontend/devops-repository/src/views/repoGeneric/index.vue b/src/frontend/devops-repository/src/views/repoGeneric/index.vue
index f6e4310256..b37211a745 100644
--- a/src/frontend/devops-repository/src/views/repoGeneric/index.vue
+++ b/src/frontend/devops-repository/src/views/repoGeneric/index.vue
@@ -320,13 +320,13 @@
created () {
this.getRepoListAll({ projectId: this.projectId })
this.initTree()
+ this.debounceClickTreeNode = debounce(this.clickTreeNodeHandler, 100)
this.pathChange()
window.repositoryVue.$on('upload-refresh', debounce((path) => {
if (path.replace(/\/[^/]+$/, '').includes(this.selectedTreeNode.fullPath)) {
this.itemClickHandler(this.selectedTreeNode)
}
}))
- this.debounceClickTreeNode = debounce(this.clickTreeNodeHandler, 100)
if (!this.community || SHOW_ANALYST_MENU) {
this.refreshSupportFileNameExtList()
}
@@ -421,17 +421,36 @@
pathChange () {
const paths = (this.$route.query.path || '').split('/').filter(Boolean)
paths.pop() // 定位到文件/文件夹的上级目录
- paths.reduce(async (chain, path) => {
- const node = await chain
- if (!node) return
- await this.updateGenericTreeNode(node)
- const child = node.children.find(child => child.name === path)
- if (!child) return
- this.sideTreeOpenList.push(child.roadMap)
- return child
- }, Promise.resolve(this.genericTree[0])).then(node => {
- this.itemClickHandler(node || this.genericTree[0])
- })
+ const tempPaths = paths
+ const num = paths.length
+ let tempTree = this.genericTree[0]
+ if (tempTree.children.length === 0 && num > 0) {
+ paths.reduce(async (chain, path) => {
+ const node = await chain
+ if (!node) return
+ await this.updateGenericTreeNode(node)
+ const child = node.children.find(child => child.name === path)
+ if (!child) return
+ this.sideTreeOpenList.push(child.roadMap)
+ return child
+ }, Promise.resolve(this.genericTree[0])).then(node => {
+ this.itemClickHandler(node || this.genericTree[0])
+ })
+ } else {
+ let destNode
+ while (tempPaths.length !== 0) {
+ tempTree = tempTree.children.find(node => node.name === tempPaths[0])
+ if (tempPaths.length === 1) {
+ destNode = tempTree
+ }
+ tempPaths.shift()
+ }
+ if (num !== 0) {
+ this.itemClickHandler(destNode)
+ } else {
+ this.itemClickHandler(this.genericTree[0])
+ }
+ }
},
// 获取中间列表数据
async getArtifactories () {
@@ -560,29 +579,31 @@
this.debounceClickTreeNode(node)
},
clickTreeNodeHandler (node) {
- this.selectedTreeNode = node
-
- this.handlerPaginationChange()
- // 更新已展开文件夹数据
- const reg = new RegExp(`^${node.roadMap}`)
- const openList = this.sideTreeOpenList
- openList.splice(0, openList.length, ...openList.filter(v => !reg.test(v)))
- // 打开选中节点的左侧树的所有祖先节点
- node.roadMap.split(',').forEach((v, i, arr) => {
- const roadMap = arr.slice(0, i + 1).join(',')
- !openList.includes(roadMap) && openList.push(roadMap)
- })
- // 更新子文件夹
- if (node.loading) return
- this.updateGenericTreeNode(node)
+ if (node.fullPath + '/default' === this.$route.query.path) {
+ this.selectedTreeNode = node
- // 更新url参数
- this.$router.replace({
- query: {
- ...this.$route.query,
- path: `${node.fullPath}/default`
- }
- })
+ this.handlerPaginationChange()
+ // 更新已展开文件夹数据
+ const reg = new RegExp(`^${node.roadMap}`)
+ const openList = this.sideTreeOpenList
+ openList.splice(0, openList.length, ...openList.filter(v => !reg.test(v)))
+ // 打开选中节点的左侧树的所有祖先节点
+ node.roadMap.split(',').forEach((v, i, arr) => {
+ const roadMap = arr.slice(0, i + 1).join(',')
+ !openList.includes(roadMap) && openList.push(roadMap)
+ })
+ // 更新子文件夹
+ if (node.loading) return
+ this.updateGenericTreeNode(node)
+ } else {
+ // 更新url参数
+ this.$router.replace({
+ query: {
+ ...this.$route.query,
+ path: `${node.fullPath}/default`
+ }
+ })
+ }
},
iconClickHandler (node) {
// 更新已展开文件夹数据
diff --git a/src/frontend/devops-repository/src/views/repoSearch/index.vue b/src/frontend/devops-repository/src/views/repoSearch/index.vue
index 5decee5f63..f7cda98965 100644
--- a/src/frontend/devops-repository/src/views/repoSearch/index.vue
+++ b/src/frontend/devops-repository/src/views/repoSearch/index.vue
@@ -119,7 +119,9 @@
},
resultList: [],
hasNext: true,
- focusContent: this.$t('toggle')
+ focusContent: this.$t('toggle'),
+ repoNames: [],
+ init: false
}
},
computed: {
@@ -180,6 +182,7 @@
projectId: this.projectId,
repoType: this.repoType,
repoName: this.repoName,
+ repoNames: this.repoNames,
packageName: this.packageName,
property: this.property,
direction: this.direction,
@@ -198,8 +201,7 @@
handlerPaginationChange ({ current = 1, limit = this.pagination.limit } = {}, scrollLoad = false) {
this.pagination.current = current
this.pagination.limit = limit
- this.searckPackageHandler(scrollLoad)
- !scrollLoad && this.$refs.infiniteScroll && this.$refs.infiniteScroll.scrollToTop()
+ this.changeQuery(scrollLoad)
},
changeSortType () {
this.refreshRoute()
@@ -290,6 +292,27 @@
permits: '',
time: 7
})
+ },
+ // ci模式下generic的查询repoName的NIN条件会和repoType的In组装条件会异常（更改为传递查询repoName，去掉repoType传递）
+ changeQuery (scrollLoad) {
+ if (this.repoType === 'generic' && MODE_CONFIG === 'ci' && !this.init) {
+ this.searchRepoList({
+ projectId: this.projectId,
+ repoType: this.repoType,
+ packageName: this.packageName || ''
+ }).then(([item]) => {
+ item.repos.forEach(item => {
+ this.repoNames.push(item.repoName)
+ })
+ this.init = true
+ this.searckPackageHandler(scrollLoad)
+ !scrollLoad && this.$refs.infiniteScroll && this.$refs.infiniteScroll.scrollToTop()
+ })
+ } else {
+ this.init = true
+ this.searckPackageHandler(scrollLoad)
+ !scrollLoad && this.$refs.infiniteScroll && this.$refs.infiniteScroll.scrollToTop()
+ }
}
}
}
From 66b48bc84abe4010daf6a7c3d55ed31f40e7c837 Mon Sep 17 00:00:00 2001
From: kunlongli <16629885+cnlkl@users.noreply.github.com>
Date: Mon, 20 Nov 2023 14:37:36 +0800
Subject: [PATCH 21/74] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E9=A1=B9?=
=?UTF-8?q?=E7=9B=AE=E6=B5=81=E9=87=8F=E4=BD=BF=E7=94=A8=E6=83=85=E5=86=B5?=
=?UTF-8?q?=E7=BB=9F=E8=AE=A1=20#1311=20(#1313)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../configuration/ScannerConfiguration.kt | 11 +
.../impl/ProjectUsageStatisticsServiceImpl.kt | 54 +++++
.../bkrepo/common/api/constant/Constants.kt | 1 +
.../listener/ArtifactTransferListener.kt | 18 +-
.../api/ProjectUsageStatisticsService.kt | 61 ++++++
.../api/pojo/ProjectUsageStatistics.kt | 43 ++++
.../pojo/ProjectUsageStatisticsListOption.kt | 45 ++++
.../service/OperateAutoConfiguration.kt | 37 +++-
.../ProjectUsageStatisticsProperties.kt | 47 +++++
.../service/dao/ProjectUsageStatisticsDao.kt | 96 +++++++++
.../ProjectUsageStatisticsInterceptor.kt | 60 ++++++
.../service/model/TProjectUsageStatistics.kt | 58 +++++
.../ProjectUsageStatisticsServiceImpl.kt | 198 ++++++++++++++++++
.../batch/ProjectUsageStatisticsCleanupJob.kt | 64 ++++++
...jectUsageStatisticsCleanupJobProperties.kt | 39 ++++
.../opdata/controller/ProjectController.kt | 24 ++-
.../api/ProjectUsageStatisticsClient.kt | 51 +++++
.../ProjectUsageStatisticsController.kt | 44 ++++
18 files changed, 944 insertions(+), 7 deletions(-)
create mode 100644 src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ProjectUsageStatisticsServiceImpl.kt
create mode 100644 src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/ProjectUsageStatisticsService.kt
create mode 100644 src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/pojo/ProjectUsageStatistics.kt
create mode 100644 src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/pojo/ProjectUsageStatisticsListOption.kt
create mode 100644 src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/config/ProjectUsageStatisticsProperties.kt
create mode 100644 src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/dao/ProjectUsageStatisticsDao.kt
create mode 100644 src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/interceptor/ProjectUsageStatisticsInterceptor.kt
create mode 100644 src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TProjectUsageStatistics.kt
create mode 100644 src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/service/ProjectUsageStatisticsServiceImpl.kt
create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectUsageStatisticsCleanupJob.kt
create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectUsageStatisticsCleanupJobProperties.kt
create mode 100644 src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/ProjectUsageStatisticsClient.kt
create mode 100644 src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/ProjectUsageStatisticsController.kt
diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt
index cbdada6416..6356371976 100644
--- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt
+++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt
@@ -33,8 +33,11 @@ import com.tencent.bkrepo.analyst.dispatcher.SubtaskPoller
import com.tencent.bkrepo.analyst.service.ExecutionClusterService
import com.tencent.bkrepo.analyst.service.ScannerService
import com.tencent.bkrepo.analyst.service.impl.OperateLogServiceImpl
+import com.tencent.bkrepo.analyst.service.impl.ProjectUsageStatisticsServiceImpl
import com.tencent.bkrepo.common.operate.api.OperateLogService
+import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
import com.tencent.bkrepo.common.service.condition.ConditionalOnNotAssembly
+import com.tencent.bkrepo.repository.api.ProjectUsageStatisticsClient
import com.tencent.bkrepo.repository.api.OperateLogClient
import org.springframework.beans.factory.ObjectProvider
import org.springframework.boot.context.properties.EnableConfigurationProperties
@@ -65,4 +68,12 @@ class ScannerConfiguration {
fun operateLogService(operateLogClient: OperateLogClient): OperateLogService {
return OperateLogServiceImpl(operateLogClient)
}
+
+ @Bean
+ @ConditionalOnNotAssembly // 仅在非单体包部署时创建，避免循环依赖问题
+ fun projectUsageStatisticsService(
+ client: ObjectProvider
+ ): ProjectUsageStatisticsService {
+ return ProjectUsageStatisticsServiceImpl(client)
+ }
}
diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ProjectUsageStatisticsServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ProjectUsageStatisticsServiceImpl.kt
new file mode 100644
index 0000000000..f7c5692ee4
--- /dev/null
+++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ProjectUsageStatisticsServiceImpl.kt
@@ -0,0 +1,54 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.analyst.service.impl
+
+import com.tencent.bkrepo.common.api.pojo.Page
+import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatistics
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatisticsListOption
+import com.tencent.bkrepo.repository.api.ProjectUsageStatisticsClient
+import org.springframework.beans.factory.ObjectProvider
+import org.springframework.scheduling.annotation.Async
+
+open class ProjectUsageStatisticsServiceImpl(
+ private val projectUsageStatisticsClient: ObjectProvider
+) : ProjectUsageStatisticsService {
+
+ @Async
+ override fun inc(projectId: String, reqCount: Long, receivedBytes: Long, responseBytes: Long) {
+ projectUsageStatisticsClient.ifAvailable?.inc(projectId, reqCount, receivedBytes, responseBytes)
+ }
+
+ override fun page(options: ProjectUsageStatisticsListOption): Page {
+ throw UnsupportedOperationException()
+ }
+
+ override fun delete(start: Long?, end: Long) {
+ throw UnsupportedOperationException()
+ }
+}
diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt
index 219cdffbb8..82d1b4b1f7 100644
--- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt
+++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/Constants.kt
@@ -88,6 +88,7 @@ const val OCI_SERVICE_NAME = "\${service.prefix:}docker\${service.suffix:}"
const val JOB_SERVICE_NAME = "\${service.prefix:}job\${service.suffix:}"
const val FS_SERVER_SERVICE_NAME = "\${service.prefix:}fs-server\${service.suffix:}"
const val MAVEN_SERVICE_NAME = "\${service.prefix:}maven\${service.suffix:}"
+const val OPDATA_SERVICE_NAME = "\${service.prefix:}opdata\${service.suffix:}"
/**
* 认证相关
diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt
index be9d1f6646..c3ba3dae40 100644
--- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt
+++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt
@@ -44,8 +44,11 @@ import com.tencent.bkrepo.common.artifact.metrics.InfluxMetricsExporter
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.stream.FileArtifactInputStream
+import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
+import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.common.service.actuator.CommonTagProvider
import com.tencent.bkrepo.common.service.util.HttpContextHolder
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.ObjectProvider
import org.springframework.context.event.EventListener
@@ -64,7 +67,8 @@ import java.util.concurrent.LinkedBlockingQueue
class ArtifactTransferListener(
private val influxMetricsExporter: ObjectProvider,
private val artifactMetricsProperties: ArtifactMetricsProperties,
- private val commonTagProvider: ObjectProvider
+ private val commonTagProvider: ObjectProvider,
+ private val projectUsageStatisticsService: ProjectUsageStatisticsService,
) {
private var queue = LinkedBlockingQueue(QUEUE_LIMIT)
@@ -76,6 +80,7 @@ class ArtifactTransferListener(
val repositoryDetail = ArtifactContextHolder.getRepoDetailOrNull()
val clientIp: String = HttpContextHolder.getClientAddress()
+ val projectId = repositoryDetail?.projectId ?: UNKNOWN
val record = ArtifactTransferRecord(
time = Instant.now(),
type = ArtifactTransferRecord.RECEIVE,
@@ -84,10 +89,13 @@ class ArtifactTransferListener(
average = throughput.average(),
storage = storageCredentials?.key ?: DEFAULT_STORAGE_KEY,
sha256 = artifactFile.getFileSha256(),
- project = repositoryDetail?.projectId ?: UNKNOWN,
+ project = projectId,
repoName = repositoryDetail?.name ?: UNKNOWN,
clientIp = clientIp
)
+ if (SecurityUtils.getUserId() != SYSTEM_USER) {
+ projectUsageStatisticsService.inc(projectId = projectId, receivedBytes = throughput.bytes)
+ }
if (artifactMetricsProperties.collectByLog) {
logger.info(
toJson(
@@ -110,6 +118,7 @@ class ArtifactTransferListener(
val repositoryDetail = ArtifactContextHolder.getRepoDetailOrNull()
val clientIp: String = HttpContextHolder.getClientAddress()
+ val projectId = repositoryDetail?.projectId ?: UNKNOWN
val record = ArtifactTransferRecord(
time = Instant.now(),
type = ArtifactTransferRecord.RESPONSE,
@@ -118,10 +127,13 @@ class ArtifactTransferListener(
average = throughput.average(),
storage = storageCredentials?.key ?: DEFAULT_STORAGE_KEY,
sha256 = artifactResource.node?.sha256.orEmpty(),
- project = repositoryDetail?.projectId ?: UNKNOWN,
+ project = projectId,
repoName = repositoryDetail?.name ?: UNKNOWN,
clientIp = clientIp
)
+ if (SecurityUtils.getUserId() != SYSTEM_USER) {
+ projectUsageStatisticsService.inc(projectId = projectId, responseBytes = throughput.bytes)
+ }
ArtifactMetrics.getDownloadedDistributionSummary().record(throughput.bytes.toDouble())
recordAccessTimeDistribution(artifactResource)
if (artifactMetricsProperties.collectByLog) {
diff --git a/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/ProjectUsageStatisticsService.kt b/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/ProjectUsageStatisticsService.kt
new file mode 100644
index 0000000000..d596422a4e
--- /dev/null
+++ b/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/ProjectUsageStatisticsService.kt
@@ -0,0 +1,61 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.operate.api
+
+import com.tencent.bkrepo.common.api.pojo.Page
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatistics
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatisticsListOption
+
+interface ProjectUsageStatisticsService {
+ /**
+ * 增加上传次数
+ *
+ * @param projectId 项目ID
+ * @param reqCount 增加的请求次数
+ * @param receivedBytes 增加上传流量大小
+ * @param responseBytes 增加下载流量大小
+ */
+ fun inc(projectId: String, reqCount: Long = 0L, receivedBytes: Long = 0L, responseBytes: Long = 0L)
+
+ /**
+ * 获取项目使用情况列表
+ *
+ * @param options 查询参数
+ *
+ * @return 项目使用情况统计数据
+ */
+ fun page(options: ProjectUsageStatisticsListOption): Page
+
+ /**
+ * 删除指定时间范围内的数据数据
+ *
+ * @param start 起始时间， 包含
+ * @param end 结束时间，不包含
+ */
+ fun delete(start: Long? = null, end: Long)
+}
diff --git a/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/pojo/ProjectUsageStatistics.kt b/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/pojo/ProjectUsageStatistics.kt
new file mode 100644
index 0000000000..73d1605e82
--- /dev/null
+++ b/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/pojo/ProjectUsageStatistics.kt
@@ -0,0 +1,43 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.operate.api.pojo
+
+import io.swagger.annotations.ApiModel
+import io.swagger.annotations.ApiModelProperty
+
+@ApiModel("项目使用情况统计数据")
+data class ProjectUsageStatistics(
+ @ApiModelProperty("项目ID")
+ val projectId: String,
+ @ApiModelProperty("请求次数")
+ val reqCount: Long = 0L,
+ @ApiModelProperty("项目上传流量")
+ var receiveBytes: Long = 0L,
+ @ApiModelProperty("项目下载流量")
+ var responseBytes: Long = 0L,
+)
diff --git a/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/pojo/ProjectUsageStatisticsListOption.kt b/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/pojo/ProjectUsageStatisticsListOption.kt
new file mode 100644
index 0000000000..7384a3ab22
--- /dev/null
+++ b/src/backend/common/common-operate/operate-api/src/main/kotlin/com/tencent/bkrepo/common/operate/api/pojo/ProjectUsageStatisticsListOption.kt
@@ -0,0 +1,45 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.operate.api.pojo
+
+import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER
+import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE
+import io.swagger.annotations.ApiModelProperty
+
+data class ProjectUsageStatisticsListOption(
+ @ApiModelProperty("项目ID")
+ val projectId: String? = null,
+ @ApiModelProperty("开始时间，包含")
+ val start: Long? = null,
+ @ApiModelProperty("结束时间，不包含")
+ val end: Long? = null,
+ @ApiModelProperty("分页数")
+ val pageNumber: Int = DEFAULT_PAGE_NUMBER,
+ @ApiModelProperty("分页大小")
+ val pageSize: Int = DEFAULT_PAGE_SIZE,
+)
diff --git a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/OperateAutoConfiguration.kt b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/OperateAutoConfiguration.kt
index 4df49c8a8f..12e1c73ece 100644
--- a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/OperateAutoConfiguration.kt
+++ b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/OperateAutoConfiguration.kt
@@ -30,24 +30,32 @@ package com.tencent.bkrepo.common.operate.service
import com.tencent.bkrepo.common.api.pojo.ClusterArchitecture
import com.tencent.bkrepo.common.api.pojo.ClusterNodeType
import com.tencent.bkrepo.common.operate.api.OperateLogService
+import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
import com.tencent.bkrepo.common.operate.service.aop.LogOperateAspect
import com.tencent.bkrepo.common.operate.service.config.OperateProperties
+import com.tencent.bkrepo.common.operate.service.config.ProjectUsageStatisticsProperties
import com.tencent.bkrepo.common.operate.service.dao.OperateLogDao
+import com.tencent.bkrepo.common.operate.service.dao.ProjectUsageStatisticsDao
+import com.tencent.bkrepo.common.operate.service.interceptor.ProjectUsageStatisticsInterceptor
import com.tencent.bkrepo.common.operate.service.service.CommitEdgeOperateLogServiceImpl
import com.tencent.bkrepo.common.operate.service.service.OperateLogServiceImpl
+import com.tencent.bkrepo.common.operate.service.service.ProjectUsageStatisticsServiceImpl
import com.tencent.bkrepo.common.security.manager.PermissionManager
import com.tencent.bkrepo.common.service.cluster.ClusterProperties
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication
import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.context.annotation.Import
+import org.springframework.web.servlet.config.annotation.InterceptorRegistry
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer
@Configuration
-@EnableConfigurationProperties(OperateProperties::class)
+@EnableConfigurationProperties(OperateProperties::class, ProjectUsageStatisticsProperties::class)
@ConditionalOnWebApplication
-@Import(OperateLogDao::class)
+@Import(OperateLogDao::class, ProjectUsageStatisticsDao::class)
class OperateAutoConfiguration {
@Bean
@@ -67,8 +75,33 @@ class OperateAutoConfiguration {
}
}
+ @Bean
+ @ConditionalOnMissingBean
+ fun projectUsageStatisticsService(
+ properties: ProjectUsageStatisticsProperties,
+ projectUsageStatisticsDao: ProjectUsageStatisticsDao,
+ ): ProjectUsageStatisticsService {
+ return ProjectUsageStatisticsServiceImpl(properties, projectUsageStatisticsDao)
+ }
+
@Bean
fun operateLogAspect(operateLogService: OperateLogService): LogOperateAspect {
return LogOperateAspect(operateLogService)
}
+
+ @Bean
+ @ConditionalOnWebApplication
+ @ConditionalOnProperty(value = ["project-usage-statistics.enableReqCountStatistic"])
+ fun projectUsageStatisticsInterceptorRegister(
+ properties: ProjectUsageStatisticsProperties,
+ service: ProjectUsageStatisticsService,
+ ): WebMvcConfigurer {
+ return object : WebMvcConfigurer {
+ override fun addInterceptors(registry: InterceptorRegistry) {
+ // 不统计服务间调用
+ registry.addInterceptor(ProjectUsageStatisticsInterceptor(properties, service))
+ .excludePathPatterns("/service/**", "/replica/**")
+ }
+ }
+ }
}
diff --git a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/config/ProjectUsageStatisticsProperties.kt b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/config/ProjectUsageStatisticsProperties.kt
new file mode 100644
index 0000000000..bb37ed46d5
--- /dev/null
+++ b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/config/ProjectUsageStatisticsProperties.kt
@@ -0,0 +1,47 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.operate.service.config
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+@ConfigurationProperties(prefix = "project-usage-statistics")
+data class ProjectUsageStatisticsProperties(
+ var enabled: Boolean = false,
+ /**
+ * 是否统计项目接口请求次数
+ */
+ var enableReqCountStatistic: Boolean = false,
+ /**
+ * 需要忽略项目ID前缀
+ */
+ var ignoredProjectIdPrefix: Set = emptySet(),
+ /**
+ * 缓存刷入数据库限流默认为100ops
+ */
+ var flushRateLimit: Double = 100.0,
+)
diff --git a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/dao/ProjectUsageStatisticsDao.kt b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/dao/ProjectUsageStatisticsDao.kt
new file mode 100644
index 0000000000..6d7b58a7a9
--- /dev/null
+++ b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/dao/ProjectUsageStatisticsDao.kt
@@ -0,0 +1,96 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.operate.service.dao
+
+import com.mongodb.client.result.DeleteResult
+import com.tencent.bkrepo.common.api.pojo.Page
+import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao
+import com.tencent.bkrepo.common.mongo.dao.util.Pages
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatisticsListOption
+import com.tencent.bkrepo.common.operate.service.model.TProjectUsageStatistics
+import org.springframework.data.mongodb.core.query.Criteria
+import org.springframework.data.mongodb.core.query.Query
+import org.springframework.data.mongodb.core.query.Update
+import org.springframework.data.mongodb.core.query.isEqualTo
+import org.springframework.stereotype.Repository
+
+@Repository
+class ProjectUsageStatisticsDao : SimpleMongoDao() {
+ fun inc(
+ projectId: String,
+ reqCount: Long,
+ receivedBytes: Long,
+ responseBytes: Long,
+ start: Long,
+ ) {
+ val criteria = TProjectUsageStatistics::start.isEqualTo(start)
+ .and(TProjectUsageStatistics::projectId.name).isEqualTo(projectId)
+ var needToUpdate = false
+ val update = Update()
+ if (reqCount != 0L) {
+ update.inc(TProjectUsageStatistics::reqCount.name, reqCount)
+ needToUpdate = true
+ }
+
+ if (receivedBytes != 0L) {
+ update.inc(TProjectUsageStatistics::receiveBytes.name, receivedBytes)
+ needToUpdate = true
+ }
+
+ if (responseBytes != 0L) {
+ update.inc(TProjectUsageStatistics::responseByte.name, responseBytes)
+ needToUpdate = true
+ }
+
+ if (needToUpdate) {
+ upsert(Query(criteria), update)
+ }
+ }
+
+ fun find(options: ProjectUsageStatisticsListOption): Page {
+ with(options) {
+ val criteria = Criteria()
+ projectId?.let { criteria.and(TProjectUsageStatistics::projectId.name).isEqualTo(it) }
+ start?.let { criteria.and(TProjectUsageStatistics::start.name).gte(it) }
+ end?.let { criteria.and(TProjectUsageStatistics::start.name).lt(it) }
+ val query = Query(criteria)
+ val total = count(query)
+ val pageRequest = Pages.ofRequest(pageNumber, pageSize)
+ query.with(pageRequest)
+ val records = find(query)
+ return Pages.ofResponse(pageRequest, total, records)
+ }
+ }
+
+ fun delete(start: Long?, end: Long): DeleteResult {
+ val criteria = Criteria()
+ start?.let { criteria.and(TProjectUsageStatistics::start.name).gte(it) }
+ criteria.and(TProjectUsageStatistics::start.name).lt(end)
+ return remove(Query(criteria))
+ }
+}
diff --git a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/interceptor/ProjectUsageStatisticsInterceptor.kt b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/interceptor/ProjectUsageStatisticsInterceptor.kt
new file mode 100644
index 0000000000..1a166080cc
--- /dev/null
+++ b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/interceptor/ProjectUsageStatisticsInterceptor.kt
@@ -0,0 +1,60 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.operate.service.interceptor
+
+import com.tencent.bkrepo.common.artifact.constant.PROJECT_ID
+import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
+import com.tencent.bkrepo.common.operate.service.config.ProjectUsageStatisticsProperties
+import com.tencent.bkrepo.common.security.util.SecurityUtils
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+import org.springframework.http.HttpMethod
+import org.springframework.web.servlet.HandlerInterceptor
+import org.springframework.web.servlet.HandlerMapping
+import javax.servlet.http.HttpServletRequest
+import javax.servlet.http.HttpServletResponse
+
+open class ProjectUsageStatisticsInterceptor(
+ private val properties: ProjectUsageStatisticsProperties,
+ private val service: ProjectUsageStatisticsService,
+) : HandlerInterceptor {
+ override fun preHandle(request: HttpServletRequest, response: HttpServletResponse, handler: Any): Boolean {
+ val enabled = properties.enabled && properties.enableReqCountStatistic
+ if (!enabled || SecurityUtils.getUserId() == SYSTEM_USER || request.method == HttpMethod.HEAD.name) {
+ return true
+ }
+
+ getProjectId(request)?.let { service.inc(projectId = it, reqCount = 1) }
+ return true
+ }
+
+ private fun getProjectId(request: HttpServletRequest): String? {
+ val uriAttribute = request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE) ?: return null
+ require(uriAttribute is Map<*, *>)
+ return uriAttribute[PROJECT_ID]?.toString()
+ }
+}
diff --git a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TProjectUsageStatistics.kt b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TProjectUsageStatistics.kt
new file mode 100644
index 0000000000..0ef1ae093e
--- /dev/null
+++ b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TProjectUsageStatistics.kt
@@ -0,0 +1,58 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.operate.service.model
+
+import org.springframework.data.mongodb.core.index.CompoundIndex
+import org.springframework.data.mongodb.core.index.CompoundIndexes
+import org.springframework.data.mongodb.core.mapping.Document
+
+@Document("project_usage_statistics")
+@CompoundIndexes(
+ CompoundIndex(name = "start_idx", def = "{'start':1}", background = true),
+ CompoundIndex(name = "projectId_start_idx", def = "{'projectId':1, 'start':1}", unique = true, background = true)
+)
+data class TProjectUsageStatistics(
+ var id: String? = null,
+ var projectId: String,
+ /**
+ * 统计周期起始时间戳
+ */
+ var start: Long,
+ /**
+ * api请求次数
+ */
+ var reqCount: Long = 0L,
+ /**
+ * 项目上传流量
+ */
+ var receiveBytes: Long = 0L,
+ /**
+ * 项目下载流量
+ */
+ var responseByte: Long = 0L,
+)
diff --git a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/service/ProjectUsageStatisticsServiceImpl.kt b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/service/ProjectUsageStatisticsServiceImpl.kt
new file mode 100644
index 0000000000..dc632ae1e9
--- /dev/null
+++ b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/service/ProjectUsageStatisticsServiceImpl.kt
@@ -0,0 +1,198 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.operate.service.service
+
+import com.google.common.cache.CacheBuilder
+import com.google.common.cache.CacheLoader
+import com.google.common.cache.LoadingCache
+import com.google.common.util.concurrent.RateLimiter
+import com.google.common.util.concurrent.ThreadFactoryBuilder
+import com.tencent.bkrepo.common.api.pojo.Page
+import com.tencent.bkrepo.common.mongo.dao.util.Pages
+import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatistics
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatisticsListOption
+import com.tencent.bkrepo.common.operate.service.config.ProjectUsageStatisticsProperties
+import com.tencent.bkrepo.common.operate.service.dao.ProjectUsageStatisticsDao
+import com.tencent.bkrepo.common.operate.service.model.TProjectUsageStatistics
+import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace
+import org.slf4j.LoggerFactory
+import org.springframework.scheduling.annotation.Scheduled
+import java.time.LocalDate
+import java.time.ZoneId
+import java.util.concurrent.ArrayBlockingQueue
+import java.util.concurrent.RejectedExecutionHandler
+import java.util.concurrent.ThreadPoolExecutor
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.LongAdder
+import javax.annotation.PreDestroy
+
+open class ProjectUsageStatisticsServiceImpl(
+ private val properties: ProjectUsageStatisticsProperties,
+ private val projectUsageStatisticsDao: ProjectUsageStatisticsDao,
+) : ProjectUsageStatisticsService {
+
+ /**
+ * 对flush操作进行限流，避免造成数据库高负载
+ */
+ private val rateLimiter = RateLimiter.create(properties.flushRateLimit)
+
+ private val executor = ThreadPoolExecutor(
+ 4,
+ 8,
+ 60,
+ TimeUnit.SECONDS,
+ ArrayBlockingQueue(10000),
+ ThreadFactoryBuilder().setNameFormat("update-project-statistics-%d").build(),
+ RejectedExecutionHandler { r, e ->
+ if (!e.isShutdown) {
+ logger.warn("caller runs update project statistics")
+ r.run()
+ }
+ }
+ )
+
+ private val cache: LoadingCache = CacheBuilder.newBuilder()
+ .maximumSize(10000)
+ .concurrencyLevel(1)
+ .expireAfterWrite(30L, TimeUnit.MINUTES)
+ .removalListener {
+ executor.execute(
+ Runnable {
+ rateLimiter.acquire()
+ synchronized(it.value!!) {
+ flush(it.key!!, it.value!!)
+ }
+ }.trace()
+ )
+ }
+ .build(CacheLoader.from { _ -> ProjectUsageStatisticsAdder() })
+
+ override fun inc(projectId: String, reqCount: Long, receivedBytes: Long, responseBytes: Long) {
+ if (!properties.enabled) {
+ return
+ }
+
+ // 不统计指定前缀的项目
+ properties.ignoredProjectIdPrefix.forEach {
+ if (projectId.startsWith(it)) {
+ return
+ }
+ }
+
+ while (true) {
+ val adder = cache.get(projectId)
+ var added = false
+ // 加锁避免inc与flush操作同一个adder对象导致丢失部分计数
+ synchronized(adder) {
+ if (!adder.flushed) {
+ adder.reqCount.add(reqCount)
+ adder.receivedBytes.add(receivedBytes)
+ adder.responseBytes.add(responseBytes)
+ added = true
+ }
+ }
+ if (added) {
+ break
+ }
+ logger.warn("adder of project[$projectId] was flushed, try to get new adder")
+ }
+ }
+
+ override fun page(options: ProjectUsageStatisticsListOption): Page {
+ val page = projectUsageStatisticsDao.find(options)
+ return Pages.buildPage(page.records.map { it.convert() }, page.pageNumber, page.pageSize)
+ }
+
+ override fun delete(start: Long?, end: Long) {
+ projectUsageStatisticsDao.delete(start, end)
+ }
+
+ @PreDestroy
+ open fun destroy() {
+ if (!properties.enabled) {
+ return
+ }
+
+ logger.info("${cache.size()} project will update usage statistics")
+ cache.invalidateAll()
+ executor.shutdown()
+ val waitMinutes = 3L
+ if (executor.awaitTermination(waitMinutes, TimeUnit.MINUTES)) {
+ logger.info("${cache.size()} project update usage statistics finished")
+ } else {
+ logger.error(
+ "cache flush executor termination timeout after $waitMinutes minutes, " +
+ "cache size[${cache.size()}, " +
+ "executor active[${executor.activeCount}, queue[${executor.queue.size}]]]"
+ )
+ }
+ }
+
+ @Scheduled(cron = "0 55 23 * * ?")
+ open fun flush() {
+ if (properties.enabled) {
+ logger.info("try to flush all projects usage statistics")
+ cache.invalidateAll()
+ }
+ }
+
+ private fun flush(projectId: String, adder: ProjectUsageStatisticsAdder) {
+ val start = LocalDate.now().atStartOfDay().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()
+ projectUsageStatisticsDao.inc(
+ projectId,
+ adder.reqCount.toLong(),
+ adder.receivedBytes.toLong(),
+ adder.responseBytes.toLong(),
+ start
+ )
+ adder.flushed = true
+ }
+
+ private fun TProjectUsageStatistics.convert(): ProjectUsageStatistics = ProjectUsageStatistics(
+ projectId = projectId,
+ reqCount = reqCount,
+ receiveBytes = receiveBytes,
+ responseBytes = responseByte,
+ )
+
+ private data class ProjectUsageStatisticsAdder(
+ val reqCount: LongAdder = LongAdder(),
+ val receivedBytes: LongAdder = LongAdder(),
+ val responseBytes: LongAdder = LongAdder(),
+ /**
+ * 是否已经写入缓存
+ */
+ @Volatile
+ var flushed: Boolean = false
+ )
+
+ companion object {
+ private val logger = LoggerFactory.getLogger(ProjectUsageStatisticsServiceImpl::class.java)
+ }
+}
diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectUsageStatisticsCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectUsageStatisticsCleanupJob.kt
new file mode 100644
index 0000000000..18c61791e9
--- /dev/null
+++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ProjectUsageStatisticsCleanupJob.kt
@@ -0,0 +1,64 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.batch
+
+import com.tencent.bkrepo.job.batch.base.DefaultContextJob
+import com.tencent.bkrepo.job.batch.base.JobContext
+import com.tencent.bkrepo.job.config.properties.ProjectUsageStatisticsCleanupJobProperties
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.data.mongodb.core.MongoTemplate
+import org.springframework.data.mongodb.core.query.Criteria
+import org.springframework.data.mongodb.core.query.Query
+import org.springframework.stereotype.Component
+import java.time.Duration
+import java.time.LocalDate
+import java.time.ZoneId
+
+@Component
+@EnableConfigurationProperties(ProjectUsageStatisticsCleanupJobProperties::class)
+class ProjectUsageStatisticsCleanupJob(
+ private val properties: ProjectUsageStatisticsCleanupJobProperties,
+ private val mongoTemplate: MongoTemplate,
+) : DefaultContextJob(properties) {
+
+ override fun getLockAtMostFor(): Duration = Duration.ofHours(1)
+
+ override fun doStart0(jobContext: JobContext) {
+ val beforeTimestamp = LocalDate
+ .now()
+ .minusDays(properties.keepDays)
+ .atStartOfDay(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli()
+ mongoTemplate.remove(Query(Criteria.where("start").lt(beforeTimestamp)), COLLECTION_NAME)
+ }
+
+ companion object {
+ private const val COLLECTION_NAME = "project_usage_statistics"
+ }
+}
diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectUsageStatisticsCleanupJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectUsageStatisticsCleanupJobProperties.kt
new file mode 100644
index 0000000000..cad34f5e24
--- /dev/null
+++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/ProjectUsageStatisticsCleanupJobProperties.kt
@@ -0,0 +1,39 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.config.properties
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+@ConfigurationProperties("job.project-usage-statistics-cleanup")
+class ProjectUsageStatisticsCleanupJobProperties(
+ override var cron: String = "0 0 4 * * ?",
+ /**
+ * 数据保留天数
+ */
+ var keepDays: Long = 90L,
+) : MongodbJobProperties()
diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt
index f36834c416..c06916d43b 100644
--- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt
+++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt
@@ -27,10 +27,16 @@
package com.tencent.bkrepo.opdata.controller
+import com.tencent.bkrepo.auth.pojo.enums.PermissionAction
import com.tencent.bkrepo.common.api.pojo.Page
import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatistics
+import com.tencent.bkrepo.common.operate.api.pojo.ProjectUsageStatisticsListOption
+import com.tencent.bkrepo.common.security.manager.PermissionManager
import com.tencent.bkrepo.common.security.permission.Principal
import com.tencent.bkrepo.common.security.permission.PrincipalType
+import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.common.service.util.ResponseBuilder
import com.tencent.bkrepo.opdata.model.TProjectMetrics
import com.tencent.bkrepo.opdata.pojo.ProjectMetrics
@@ -43,21 +49,35 @@ import org.springframework.web.bind.annotation.RestController
@RestController
@RequestMapping("/api/project/metrics")
-@Principal(PrincipalType.ADMIN)
class ProjectController(
- private val projectMetricsService: ProjectMetricsService
+ private val projectMetricsService: ProjectMetricsService,
+ private val projectUsageStatisticsService: ProjectUsageStatisticsService,
+ private val permissionManager: PermissionManager,
) {
/**
* 获取项目的统计数据
*/
@GetMapping("/list")
+ @Principal(PrincipalType.ADMIN)
fun getProjectMetrics(
option: ProjectMetricsOption
): Response> {
return ResponseBuilder.success(projectMetricsService.page(option))
}
+ @GetMapping("/list/usage")
+ fun getUsageStatistics(
+ options: ProjectUsageStatisticsListOption
+ ): Response> {
+ if (options.projectId == null) {
+ permissionManager.checkPrincipal(SecurityUtils.getUserId(), PrincipalType.ADMIN)
+ } else {
+ permissionManager.checkProjectPermission(PermissionAction.MANAGE, options.projectId!!)
+ }
+ return ResponseBuilder.success(projectUsageStatisticsService.page(options))
+ }
+
/**
* 获取项目的统计数据
*/
diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/ProjectUsageStatisticsClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/ProjectUsageStatisticsClient.kt
new file mode 100644
index 0000000000..78cbd08080
--- /dev/null
+++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/ProjectUsageStatisticsClient.kt
@@ -0,0 +1,51 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.repository.api
+
+import com.tencent.bkrepo.common.api.constant.REPOSITORY_SERVICE_NAME
+import com.tencent.bkrepo.common.api.pojo.Response
+import io.swagger.annotations.Api
+import io.swagger.annotations.ApiOperation
+import org.springframework.cloud.openfeign.FeignClient
+import org.springframework.web.bind.annotation.PostMapping
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
+
+@Api("操作统计数据接口")
+@FeignClient(REPOSITORY_SERVICE_NAME, contextId = "ProjectUsageStatisticsClient")
+@RequestMapping("/service/project/usage/statistics")
+interface ProjectUsageStatisticsClient {
+ @ApiOperation("增加使用数据")
+ @PostMapping("/inc")
+ fun inc(
+ @RequestParam projectId: String,
+ @RequestParam reqCount: Long,
+ @RequestParam receivedBytes: Long,
+ @RequestParam responseBytes: Long
+ ): Response
+}
diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/ProjectUsageStatisticsController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/ProjectUsageStatisticsController.kt
new file mode 100644
index 0000000000..14ec7de557
--- /dev/null
+++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/ProjectUsageStatisticsController.kt
@@ -0,0 +1,44 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.repository.controller.service
+
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.operate.api.ProjectUsageStatisticsService
+import com.tencent.bkrepo.common.service.util.ResponseBuilder
+import com.tencent.bkrepo.repository.api.ProjectUsageStatisticsClient
+import org.springframework.web.bind.annotation.RestController
+
+@RestController
+class ProjectUsageStatisticsController(
+ private val projectUsageStatisticsService: ProjectUsageStatisticsService,
+) : ProjectUsageStatisticsClient {
+ override fun inc(projectId: String, reqCount: Long, receivedBytes: Long, responseBytes: Long): Response {
+ projectUsageStatisticsService.inc(projectId, reqCount, receivedBytes, responseBytes)
+ return ResponseBuilder.success()
+ }
+}
From 9773e147bdfa079d61661a6eaab139d24060e0c3 Mon Sep 17 00:00:00 2001
From: lannoy0523 <46735290+lannoy0523@users.noreply.github.com>
Date: Mon, 20 Nov 2023 14:38:07 +0800
Subject: [PATCH 22/74] =?UTF-8?q?feat:=E6=96=B0=E5=A2=9E=E8=8A=82=E7=82=B9?=
=?UTF-8?q?=E6=B8=85=E7=90=86=20#1437=20(#1438)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat:新增节点清理 #1437
* bug:opdata 仓库大小统计页面问题修复 #1455
* Revert "bug:opdata 仓库大小统计页面问题修复 #1455"
This reverts commit 934d07fcbd2119de61c0a78ca3ff265ba7b17095.
* feat:新增节点清理,前端防止二次提交 #1437
---
.../devops-repository/src/images/right.svg | 1 +
.../src/store/actions/repoGeneric.js | 11 +++
.../devops-repository/src/utils/date.js | 10 ++
.../views/repoGeneric/genericCleanDialog.vue | 99 +++++++++++++++++++
.../src/views/repoGeneric/index.vue | 47 ++++++++-
.../src/views/repoList/index.vue | 26 ++++-
src/frontend/locale/repository/en-US.json | 9 +-
src/frontend/locale/repository/zh-CN.json | 9 +-
8 files changed, 207 insertions(+), 5 deletions(-)
create mode 100644 src/frontend/devops-repository/src/images/right.svg
create mode 100644 src/frontend/devops-repository/src/views/repoGeneric/genericCleanDialog.vue
diff --git a/src/frontend/devops-repository/src/images/right.svg b/src/frontend/devops-repository/src/images/right.svg
new file mode 100644
index 0000000000..f964cd54d2
--- /dev/null
+++ b/src/frontend/devops-repository/src/images/right.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/src/frontend/devops-repository/src/store/actions/repoGeneric.js b/src/frontend/devops-repository/src/store/actions/repoGeneric.js
index fdad233326..ddd775e663 100644
--- a/src/frontend/devops-repository/src/store/actions/repoGeneric.js
+++ b/src/frontend/devops-repository/src/store/actions/repoGeneric.js
@@ -317,5 +317,16 @@ export default {
return Vue.prototype.$ajax.get(
`${prefix}/project/metrics/${projectId}`
)
+ },
+ // 清理创建时间早于{date}的文件节点
+ cleanNode (_, { path, date }) {
+ return Vue.prototype.$ajax.delete(
+ `${prefix}/node/clean/${path}`,
+ {
+ params: {
+ date: date
+ }
+ }
+ )
}
}
diff --git a/src/frontend/devops-repository/src/utils/date.js b/src/frontend/devops-repository/src/utils/date.js
index fdc3c39021..a3588a4b5a 100644
--- a/src/frontend/devops-repository/src/utils/date.js
+++ b/src/frontend/devops-repository/src/utils/date.js
@@ -15,3 +15,13 @@ export function zeroTime (date) {
return moment([moment().year(), moment().month(), moment().date()]).toDate()
}
}
+
+// 获取当前时间前几年的date
+export function beforeYears (years) {
+ return moment().subtract(years, 'years').toDate()
+}
+
+// 获取当前时间前几月的date
+export function beforeMonths (months) {
+ return moment().subtract(months, 'months').toDate()
+}
diff --git a/src/frontend/devops-repository/src/views/repoGeneric/genericCleanDialog.vue b/src/frontend/devops-repository/src/views/repoGeneric/genericCleanDialog.vue
new file mode 100644
index 0000000000..f462972234
--- /dev/null
+++ b/src/frontend/devops-repository/src/views/repoGeneric/genericCleanDialog.vue
@@ -0,0 +1,99 @@
+
+
+
+
+