diff --git a/src/backend/boot-assembly/build.gradle.kts b/src/backend/boot-assembly/build.gradle.kts index 0bbe5613e3..dc1b5abd3b 100644 --- a/src/backend/boot-assembly/build.gradle.kts +++ b/src/backend/boot-assembly/build.gradle.kts @@ -46,6 +46,7 @@ dependencies { implementation(project(":replication:biz-replication")) implementation(project(":webhook:biz-webhook")) implementation(project(":lfs:biz-lfs")) + implementation(project(":ddc:biz-ddc")) } configurations.all { diff --git a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryType.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryType.kt index f7c5fd839b..3fc73afd39 100644 --- a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryType.kt +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryType.kt @@ -50,6 +50,7 @@ enum class RepositoryType { OCI, CONAN, LFS, + DDC, ; companion object { diff --git a/src/backend/ddc/api-ddc/build.gradle.kts b/src/backend/ddc/api-ddc/build.gradle.kts new file mode 100644 index 0000000000..2ef622f5be --- /dev/null +++ b/src/backend/ddc/api-ddc/build.gradle.kts @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +dependencies { + api(project(":common:common-artifact:artifact-service")) +} diff --git a/src/backend/ddc/api-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/Contant.kt b/src/backend/ddc/api-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/Contant.kt new file mode 100644 index 0000000000..9d812dfc22 --- /dev/null +++ b/src/backend/ddc/api-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/Contant.kt @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc + +const val NAME = "DDC" diff --git a/src/backend/ddc/api-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/extension/BlobDecompressExtension.kt b/src/backend/ddc/api-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/extension/BlobDecompressExtension.kt new file mode 100644 index 0000000000..34fec5888b --- /dev/null +++ b/src/backend/ddc/api-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/extension/BlobDecompressExtension.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.extension + +interface BlobDecompressExtension { + /** + * 解压数据 + * + * @param compressedPayload 压缩后的数据 + * @param offset 数据在[compressedPayload]中的起始位置，inclusive + * @param length 数据长度 + * @param result 解压后的数据 + * + * @return 写入[result]的数据长度 + */ + fun decompress(compressedPayload: ByteArray, offset: Int, length: Int, result: ByteArray): Int +} diff --git a/src/backend/ddc/biz-ddc/build.gradle.kts b/src/backend/ddc/biz-ddc/build.gradle.kts new file mode 100644 index 0000000000..75dc4e4a78 --- /dev/null +++ b/src/backend/ddc/biz-ddc/build.gradle.kts @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +dependencies { + implementation(project(":ddc:api-ddc")) + implementation(project(":repository:api-repository")) + implementation("org.bouncycastle:bcpkix-jdk15on:1.69") +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/CompressedBlobArtifactInfo.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/CompressedBlobArtifactInfo.kt new file mode 100644 index 0000000000..8eaed8a601 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/CompressedBlobArtifactInfo.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.artifact + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.ddc.utils.DdcUtils.DIR_BLOBS + +class CompressedBlobArtifactInfo( + projectId: String, + repoName: String, + val contentId: String, + var compressedContentId: String? = null +) : ArtifactInfo(projectId, repoName, StringPool.EMPTY) { + + override fun getArtifactName() = "/$DIR_BLOBS/$compressedContentId" + + override fun getArtifactFullPath() = "/$DIR_BLOBS/$compressedContentId" + + companion object { + const val PATH_VARIABLE_CONTENT_ID = "id" + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/DdcArtifactConfigurer.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/DdcArtifactConfigurer.kt new file mode 100644 index 0000000000..7fa51bfe9f --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/DdcArtifactConfigurer.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.artifact + +import com.tencent.bkrepo.common.artifact.config.ArtifactConfigurerSupport +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurityCustomizer +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.ddc.artifact.repository.DdcLocalRepository +import com.tencent.bkrepo.ddc.artifact.repository.DdcRemoteRepository +import com.tencent.bkrepo.ddc.artifact.repository.DdcVirtualRepository +import org.springframework.stereotype.Component + +@Component +class DdcArtifactConfigurer : ArtifactConfigurerSupport() { + + override fun getRepositoryType() = RepositoryType.DDC + override fun getLocalRepository() = SpringContextUtils.getBean() + override fun getRemoteRepository() = SpringContextUtils.getBean() + override fun getVirtualRepository() = SpringContextUtils.getBean() + + override fun getAuthSecurityCustomizer() = + HttpAuthSecurityCustomizer { httpAuthSecurity -> httpAuthSecurity.withPrefix("/ddc") } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/ReferenceArtifactInfo.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/ReferenceArtifactInfo.kt new file mode 100644 index 0000000000..66196cf528 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/ReferenceArtifactInfo.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.artifact + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.ddc.pojo.RefId + +class ReferenceArtifactInfo( + projectId: String, + repoName: String, + val bucket: String, + val refId: RefId, + var inlineBlobHash: String? = null, +) : ArtifactInfo(projectId, repoName, StringPool.EMPTY) { + + override fun getArtifactName() = "/$bucket/$refId" + + override fun getArtifactFullPath() = "/$bucket/$refId" + + companion object { + const val PATH_VARIABLE_BUCKET = "bucket" + const val PATH_VARIABLE_REF_ID = "key" + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt new file mode 100644 index 0000000000..6ba7f03bc5 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt @@ -0,0 +1,379 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.artifact.repository + +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.constant.MediaTypes +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext +import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository +import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource +import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.ddc.artifact.CompressedBlobArtifactInfo +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo +import com.tencent.bkrepo.ddc.exception.BlobNotFoundException +import com.tencent.bkrepo.ddc.exception.NotImplementedException +import com.tencent.bkrepo.ddc.exception.ReferenceIsMissingBlobsException +import com.tencent.bkrepo.ddc.pojo.Blob +import com.tencent.bkrepo.ddc.pojo.Reference +import com.tencent.bkrepo.ddc.pojo.UploadCompressedBlobResponse +import com.tencent.bkrepo.ddc.serialization.CbObject +import com.tencent.bkrepo.ddc.service.BlobService +import com.tencent.bkrepo.ddc.service.ReferenceResolver +import com.tencent.bkrepo.ddc.service.ReferenceService +import com.tencent.bkrepo.ddc.utils.BlakeUtils.blake3 +import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_JUPITER_INLINED_PAYLOAD +import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_COMPACT_BINARY +import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_UNREAL_COMPRESSED_BUFFER +import com.tencent.bkrepo.ddc.utils.NODE_METADATA_KEY_BLOB_ID +import com.tencent.bkrepo.ddc.utils.NODE_METADATA_KEY_CONTENT_ID +import com.tencent.bkrepo.ddc.utils.isAttachment +import com.tencent.bkrepo.ddc.utils.isBinaryAttachment +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component +import java.io.ByteArrayInputStream +import java.nio.ByteBuffer +import java.time.format.DateTimeFormatter + +@Component +class DdcLocalRepository( + private val referenceService: ReferenceService, + private val refResolver: ReferenceResolver, + private val blobService: BlobService, +) : LocalRepository() { + override fun onUploadBefore(context: ArtifactUploadContext) { + super.onUploadBefore(context) + var uploadBlake3: String? = null + val artifactInfo = context.artifactInfo + if (artifactInfo is ReferenceArtifactInfo) { + uploadBlake3 = artifactInfo.inlineBlobHash!! + } + if (uploadBlake3 != null && uploadBlake3 != context.getStreamArtifactFile().blake3().hex()) { + throw ErrorCodeException(ArtifactMessageCode.DIGEST_CHECK_FAILED, "blake3") + } + } + + override fun onUpload(context: ArtifactUploadContext) { + val artifactInfo = context.artifactInfo + if (artifactInfo is ReferenceArtifactInfo) { + onUploadReference(context) + } else { + onUploadBlob(context) + } + } + + override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { + val artifactInfo = context.artifactInfo + return if (artifactInfo is ReferenceArtifactInfo) { + onDownloadReference(context) + } else { + onDownloadBlob(context) + } + } + + fun finalizeRef(artifactInfo: ReferenceArtifactInfo) { + with(artifactInfo) { + val ref = getReference( + projectId, repoName, bucket, refId.toString(), checkFinalized = false + ) ?: throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "No blob when attempting to finalize") + if (ref.blobId!!.toString() != artifactInfo.inlineBlobHash) { + throw ErrorCodeException(ArtifactMessageCode.DIGEST_CHECK_FAILED, "blake3") + } + val res = referenceService.finalize(ref, ref.inlineBlob!!) + HttpContextHolder.getResponse().writer.println(res.toJsonString()) + } + } + + private fun onUploadReference(context: ArtifactUploadContext) { + val contentType = context.request.contentType + val artifactInfo = context.artifactInfo as ReferenceArtifactInfo + when (contentType) { + MEDIA_TYPE_UNREAL_COMPACT_BINARY -> { + val payload = context.getArtifactFile().getInputStream().readBytes() + val ref = referenceService.create(Reference.from(artifactInfo, payload)) + ref.inlineBlob?.let { + // inlineBlob为null时表示inlineBlob过大，需要存到文件中 + val nodeCreateRequest = buildRefNodeCreateRequest(context) + storageManager.storeArtifactFile( + nodeCreateRequest, context.getArtifactFile(), context.storageCredentials + ) + } + val res = referenceService.finalize(ref, payload) + HttpContextHolder.getResponse().writer.println(res.toJsonString()) + } + + else -> throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, "Unknown request type $contentType" + ) + } + } + + private fun buildRefNodeCreateRequest(context: ArtifactUploadContext): NodeCreateRequest { + val artifactInfo = context.artifactInfo as ReferenceArtifactInfo + val metadata = ArrayList() + metadata.add( + MetadataModel( + key = NODE_METADATA_KEY_BLOB_ID, + value = artifactInfo.inlineBlobHash!!, + system = true + ) + ) + + return buildNodeCreateRequest(context).copy( + overwrite = true, + nodeMetadata = metadata + ) + } + + private fun onUploadBlob(context: ArtifactUploadContext) { + with(context) { + val artifactInfo = artifactInfo as CompressedBlobArtifactInfo + // TODO 校验解压后blob hash与contentId是否相等 + // TODO 改为读取流时直接计算blake3，避免重复读流 + artifactInfo.compressedContentId = getStreamArtifactFile().blake3().hex() + + storageManager.storeArtifactFile(buildBlobNodeCreateRequest(context), getArtifactFile(), storageCredentials) + blobService.create(Blob.from(artifactInfo, getArtifactSha256(), getArtifactFile().getSize())) + HttpContextHolder + .getResponse() + .writer + .println(UploadCompressedBlobResponse(artifactInfo.contentId).toJsonString()) + } + } + + private fun buildBlobNodeCreateRequest(context: ArtifactUploadContext): NodeCreateRequest { + val artifactInfo = context.artifactInfo as CompressedBlobArtifactInfo + val metadata = ArrayList() + metadata.add( + MetadataModel( + key = NODE_METADATA_KEY_BLOB_ID, + value = artifactInfo.compressedContentId!!, + system = true + ) + ) + metadata.add( + MetadataModel( + key = NODE_METADATA_KEY_CONTENT_ID, + value = artifactInfo.contentId + ) + ) + + return buildNodeCreateRequest(context).copy( + overwrite = true, + nodeMetadata = metadata + ) + } + + private fun onDownloadReference(context: ArtifactDownloadContext): ArtifactResource? { + with(context) { + val artifactInfo = context.artifactInfo as ReferenceArtifactInfo + val ref = getReference( + projectId, repoName, artifactInfo.bucket, artifactInfo.refId.toString(), true + ) ?: return null + response.addHeader(HEADER_NAME_HASH, ref.blobId.toString()) + response.addHeader(HEADER_NAME_LAST_ACCESS, ref.lastAccessDate!!.format(DATE_TIME_FORMATTER)) + + return when (val responseType = response.contentType) { + MEDIA_TYPE_UNREAL_COMPACT_BINARY -> { + val ais = ArtifactInputStream( + ByteArrayInputStream(ref.inlineBlob), + Range.full(ref.inlineBlob!!.size.toLong()) + ) + ArtifactResource(ais, artifactInfo.getResponseName()).apply { contentType = responseType } + } + + MEDIA_TYPE_JUPITER_INLINED_PAYLOAD -> { + onInlineDownload(context, ref.inlineBlob!!, responseType) + } + + else -> throw NotImplementedException("Unknown expected response type $responseType") + } + } + } + + private fun onDownloadBlob(context: ArtifactDownloadContext): ArtifactResource? { + with(context) { + val artifactInfo = context.artifactInfo as CompressedBlobArtifactInfo + val acceptContentType = request.getHeaders("Accept").toList() + val blob = blobService.getSmallestBlobByContentId(projectId, repoName, artifactInfo.contentId) + ?: return null + val blobId = blob.blobId.toString() + artifactInfo.compressedContentId = blobId + + val responseType = if (blobId == artifactInfo.contentId) { + MediaTypes.APPLICATION_OCTET_STREAM + } else { + MEDIA_TYPE_UNREAL_UNREAL_COMPRESSED_BUFFER + } + + if (acceptContentType.isNotEmpty() && + !acceptContentType.contains("*/*") && + !acceptContentType.contains(responseType) + ) { + throw ErrorCodeException( + status = HttpStatus.UNSUPPORTED_MEDIA_TYPE, + messageCode = CommonMessageCode.MEDIA_TYPE_UNSUPPORTED + ) + } + + val blobInputStream = blobService.loadBlob(blob) + val resource = ArtifactResource(blobInputStream, artifactInfo.getResponseName()) + resource.contentType = responseType + return resource + } + } + + /** + * 客户端调用获取ref的接口时，直接返回缓存内容 + */ + private fun onInlineDownload( + context: ArtifactDownloadContext, + refInlineBlob: ByteArray, + responseType: String, + ): ArtifactResource? { + with(context) { + val cb = CbObject(ByteBuffer.wrap(refInlineBlob)) + val (binaryAttachmentCount, attachmentCount) = countAttachment(cb) + val artifactInfo = artifactInfo as ReferenceArtifactInfo + + // 只允许在ref只包含了单个binaryAttachment的情况下，直接通过ref接口直接获取缓存文件 + if (binaryAttachmentCount > 1 || attachmentCount > 1 || binaryAttachmentCount != attachmentCount) { + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, + "Object ${artifactInfo.bucket} ${artifactInfo.refId} had more then 1 binary attachment field," + + " unable to inline this object. Use compact object response instead." + ) + } + + return if (binaryAttachmentCount == 1) { + loadReferencedBlob(context, cb, responseType) + } else { + val range = Range.full(refInlineBlob.size.toLong()) + val ais = ArtifactInputStream(ByteArrayInputStream(refInlineBlob), range) + ArtifactResource(ais, artifactInfo.getResponseName()).apply { contentType = responseType } + } + } + } + + private fun loadReferencedBlob( + context: ArtifactDownloadContext, cb: CbObject, responseType: String + ): ArtifactResource? { + return try { + val artifactInfo = context.artifactInfo as ReferenceArtifactInfo + val blobs = refResolver.getReferencedBlobs(context.projectId, context.repoName, cb) + if (blobs.size == 1) { + blobToArtifactResource(context, blobs[0], responseType) + } else if (blobs.isEmpty()) { + null + } else { + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, + "Object ${artifactInfo.bucket} ${artifactInfo.refId} contained a content id " + + "which resolved to more then 1 blob, unable to inline this object. " + + "Use compact object response instead." + ) + } + } catch (e: ReferenceIsMissingBlobsException) { + null + } + } + + private fun blobToArtifactResource( + context: ArtifactDownloadContext, + blob: Blob, + responseType: String + ): ArtifactResource? { + with(context) { + response.addHeader(HEADER_NAME_INLINE_PAYLOAD_HASH, blob.toString()) + return try { + val blobInputStream = blobService.loadBlob(blob) + ArtifactResource(blobInputStream, artifactInfo.getResponseName()).apply { contentType = responseType } + } catch (e: BlobNotFoundException) { + null + } + } + } + + private fun getReference( + projectId: String, repoName: String, bucket: String, key: String, checkFinalized: Boolean + ): Reference? { + val ref = referenceService.getReference( + projectId, repoName, bucket, key, + includePayload = true, + checkFinalized = checkFinalized + ) ?: return null + + if (ref.inlineBlob == null) { + val repo = ArtifactContextHolder.getRepoDetail(ArtifactContextHolder.RepositoryId(projectId, repoName)) + ref.inlineBlob = nodeClient.getNodeDetail(projectId, repoName, ref.fullPath()).data?.let { + storageManager.loadArtifactInputStream(it, repo.storageCredentials)?.readBytes() + } + } + + return if (ref.inlineBlob == null) { + logger.warn("Blob was null when attempting to fetch ${ref.repoName} ${ref.bucket} ${ref.key}") + null + } else { + ref + } + } + + private fun countAttachment(cb: CbObject): Pair { + var countOfAttachmentFields = 0 + var countOfBinaryAttachmentFields = 0 + cb.iterateAttachments { + if (it.isBinaryAttachment()) { + countOfBinaryAttachmentFields++ + } + if (it.isAttachment()) { + countOfAttachmentFields++ + } + } + + return Pair(countOfBinaryAttachmentFields, countOfAttachmentFields) + } + + companion object { + private val logger = LoggerFactory.getLogger(DdcLocalRepository::class.java) + private val DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("MM/dd/yyyy hh:mm:ss") + const val HEADER_NAME_HASH = "X-Jupiter-IoHash" + private const val HEADER_NAME_LAST_ACCESS = "X-Jupiter-LastAccess" + private const val HEADER_NAME_INLINE_PAYLOAD_HASH = "X-Jupiter-InlinePayloadHash" + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcRemoteRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcRemoteRepository.kt new file mode 100644 index 0000000000..5a85e30c86 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcRemoteRepository.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.artifact.repository + +import com.tencent.bkrepo.common.artifact.repository.remote.RemoteRepository +import org.springframework.stereotype.Component + +@Component +class DdcRemoteRepository : RemoteRepository() diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcVirtualRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcVirtualRepository.kt new file mode 100644 index 0000000000..4d0547e54b --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcVirtualRepository.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.artifact.repository + +import com.tencent.bkrepo.common.artifact.repository.virtual.VirtualRepository +import org.springframework.stereotype.Component + +@Component +class DdcVirtualRepository : VirtualRepository() diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/CompressedBlobArtifactInfoResolver.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/CompressedBlobArtifactInfoResolver.kt new file mode 100644 index 0000000000..1303d354aa --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/CompressedBlobArtifactInfoResolver.kt @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.artifact.resolver + +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.resolve.path.ArtifactInfoResolver +import com.tencent.bkrepo.common.artifact.resolve.path.Resolver +import com.tencent.bkrepo.ddc.artifact.CompressedBlobArtifactInfo +import org.springframework.stereotype.Component +import org.springframework.web.servlet.HandlerMapping +import javax.servlet.http.HttpServletRequest + +@Component +@Resolver(CompressedBlobArtifactInfo::class) +class CompressedBlobArtifactInfoResolver : ArtifactInfoResolver { + override fun resolve( + projectId: String, + repoName: String, + artifactUri: String, + request: HttpServletRequest + ): ArtifactInfo { + val attributes = request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE) as Map<*, *> + return CompressedBlobArtifactInfo( + projectId = projectId, + repoName = repoName, + contentId = attributes[CompressedBlobArtifactInfo.PATH_VARIABLE_CONTENT_ID]!!.toString(), + ) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/ReferenceArtifactInfoResolver.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/ReferenceArtifactInfoResolver.kt new file mode 100644 index 0000000000..182ab7dfa8 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/ReferenceArtifactInfoResolver.kt @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.artifact.resolver + +import com.tencent.bkrepo.common.artifact.resolve.path.ArtifactInfoResolver +import com.tencent.bkrepo.common.artifact.resolve.path.Resolver +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo.Companion.PATH_VARIABLE_BUCKET +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo.Companion.PATH_VARIABLE_REF_ID +import com.tencent.bkrepo.ddc.pojo.RefId +import org.springframework.stereotype.Component +import org.springframework.web.servlet.HandlerMapping +import javax.servlet.http.HttpServletRequest + +@Component +@Resolver(ReferenceArtifactInfo::class) +class ReferenceArtifactInfoResolver : ArtifactInfoResolver { + override fun resolve( + projectId: String, + repoName: String, + artifactUri: String, + request: HttpServletRequest + ): ReferenceArtifactInfo { + val attributes = request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE) as Map<*, *> + return ReferenceArtifactInfo( + projectId = projectId, + repoName = repoName, + bucket = attributes[PATH_VARIABLE_BUCKET].toString(), + refId = RefId.create(attributes[PATH_VARIABLE_REF_ID].toString()) + ) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcConfiguration.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcConfiguration.kt new file mode 100644 index 0000000000..0472f74d5a --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcConfiguration.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.config + +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Configuration + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties(DdcProperties::class) +class DdcConfiguration diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcProperties.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcProperties.kt new file mode 100644 index 0000000000..6b941b7cb3 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/config/DdcProperties.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.config + +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.util.unit.DataSize +import org.springframework.util.unit.DataUnit + +@ConfigurationProperties("ddc") +data class DdcProperties( + var inlineBlobMaxSize: DataSize = DataSize.of(64L, DataUnit.KILOBYTES) +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/CompressedBlobController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/CompressedBlobController.kt new file mode 100644 index 0000000000..536b40e1f8 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/CompressedBlobController.kt @@ -0,0 +1,83 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.controller + +import com.tencent.bkrepo.auth.pojo.enums.PermissionAction +import com.tencent.bkrepo.auth.pojo.enums.ResourceType +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.security.permission.Permission +import com.tencent.bkrepo.ddc.artifact.CompressedBlobArtifactInfo +import com.tencent.bkrepo.ddc.service.CompressedBlobService +import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_UNREAL_COMPRESSED_BUFFER +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import org.slf4j.LoggerFactory +import org.springframework.http.MediaType +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RequestMapping("/{projectId}/api/v1/compressed-blobs") +@RestController +class CompressedBlobController( + private val compressedBlobService: CompressedBlobService +) { + + @ApiOperation("获取压缩后的缓存") + @GetMapping( + "/{repoName}/{$PATH_VARIABLE_CONTENT_ID}", + produces = [MEDIA_TYPE_UNREAL_UNREAL_COMPRESSED_BUFFER, MediaType.APPLICATION_OCTET_STREAM_VALUE] + ) + @Permission(ResourceType.REPO, action = PermissionAction.READ) + fun get( + @ApiParam(value = "ddc compressed blob", required = true) + @ArtifactPathVariable + artifactInfo: CompressedBlobArtifactInfo, + ) { + compressedBlobService.get(artifactInfo) + } + + @ApiOperation("上传压缩后的缓存") + @PutMapping("/{repoName}/{$PATH_VARIABLE_CONTENT_ID}", consumes = [MEDIA_TYPE_UNREAL_UNREAL_COMPRESSED_BUFFER]) + @Permission(ResourceType.REPO, action = PermissionAction.WRITE) + fun put( + @ApiParam(value = "ddc compressed blob", required = true) + @ArtifactPathVariable + artifactInfo: CompressedBlobArtifactInfo, + artifactFile: ArtifactFile + ) { + compressedBlobService.put(artifactInfo, artifactFile) + } + + companion object { + private val logger = LoggerFactory.getLogger(CompressedBlobController::class.java) + const val PATH_VARIABLE_CONTENT_ID = CompressedBlobArtifactInfo.PATH_VARIABLE_CONTENT_ID + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/HealthController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/HealthController.kt new file mode 100644 index 0000000000..3707ff7099 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/HealthController.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.controller + +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +class HealthController { + @GetMapping("/{projectId}/health/ready") + fun ready() { + HttpContextHolder.getResponse().apply { + status = HttpStatus.OK.value + contentType = "text/plain" + writer.write("Healthy") + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt new file mode 100644 index 0000000000..427e6818eb --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/ReferencesController.kt @@ -0,0 +1,154 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.controller + +import com.tencent.bkrepo.auth.pojo.enums.PermissionAction +import com.tencent.bkrepo.auth.pojo.enums.ResourceType +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.message.CommonMessageCode.PARAMETER_INVALID +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.security.permission.Permission +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo +import com.tencent.bkrepo.ddc.artifact.repository.DdcLocalRepository.Companion.HEADER_NAME_HASH +import com.tencent.bkrepo.ddc.service.ReferenceArtifactService +import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_JUPITER_INLINED_PAYLOAD +import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_COMPACT_BINARY +import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_COMPACT_BINARY_PACKAGE +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import org.slf4j.LoggerFactory +import org.springframework.http.MediaType +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RequestMapping("/{projectId}/api/v1/refs") +@RestController +class ReferencesController( + private val referenceArtifactService: ReferenceArtifactService +) { + + @ApiOperation("获取ref") + @Permission(ResourceType.REPO, PermissionAction.READ) + @GetMapping( + "/{repoName}/{$PATH_VARIABLE_BUCKET}/{$PATH_VARIABLE_REF_ID}", + produces = [ + MediaType.APPLICATION_JSON_VALUE, + MediaType.APPLICATION_OCTET_STREAM_VALUE, + MEDIA_TYPE_UNREAL_COMPACT_BINARY, + MEDIA_TYPE_JUPITER_INLINED_PAYLOAD, + MEDIA_TYPE_UNREAL_COMPACT_BINARY_PACKAGE + ] + ) + fun getRef( + @ApiParam(value = "ddc ref", required = true) + @ArtifactPathVariable + artifactInfo: ReferenceArtifactInfo, + ) { + HttpContextHolder.getResponse().contentType = getResponseType(null, MEDIA_TYPE_UNREAL_COMPACT_BINARY) + referenceArtifactService.downloadRef(artifactInfo) + } + + @ApiOperation("开始创建ref") + @PutMapping( + "/{repoName}/{$PATH_VARIABLE_BUCKET}/{$PATH_VARIABLE_REF_ID}", + ) + @Permission(ResourceType.REPO, PermissionAction.WRITE) + fun putObject( + @ApiParam(value = "ddc ref", required = true) + @ArtifactPathVariable + artifactInfo: ReferenceArtifactInfo, + file: ArtifactFile + ) { + artifactInfo.inlineBlobHash = HttpContextHolder.getRequest().getHeader(HEADER_NAME_HASH) + ?: throw BadRequestException(PARAMETER_INVALID, "Missing expected header $HEADER_NAME_HASH") + referenceArtifactService.createRef(artifactInfo, file) + } + + @ApiOperation("结束ref创建") + @PostMapping( + "/{repoName}/{$PATH_VARIABLE_BUCKET}/{$PATH_VARIABLE_REF_ID}/finalize/{hash}", + ) + @Permission(ResourceType.REPO, PermissionAction.WRITE) + fun finalizeObject( + @ApiParam(value = "ddc ref", required = true) + @ArtifactPathVariable + artifactInfo: ReferenceArtifactInfo, + @ApiParam("blob hash", required = true) + @PathVariable hash: String, + ) { + artifactInfo.inlineBlobHash = hash + referenceArtifactService.finalize(artifactInfo) + } + + private fun getResponseType(format: String?, default: String): String { + if (!format.isNullOrEmpty()) { + return when (format.toLowerCase()) { + "json" -> MediaType.APPLICATION_JSON_VALUE + "raw" -> MediaType.APPLICATION_OCTET_STREAM_VALUE + "uecb" -> MEDIA_TYPE_UNREAL_COMPACT_BINARY + "uecbpkg" -> MEDIA_TYPE_UNREAL_COMPACT_BINARY_PACKAGE + else -> throw BadRequestException( + PARAMETER_INVALID, + "No mapping defined from format $format to mime type" + ) + } + } + + val accept = HttpContextHolder.getRequest().getHeaders("Accept").toList() + if (accept.isEmpty() || accept.contains("*/*")) { + return default + } + + accept.firstOrNull { it.toLowerCase() in VALID_CONTENT_TYPES }?.let { return it } + + throw BadRequestException( + PARAMETER_INVALID, + "Unable to determine response type for header: $accept" + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(ReferencesController::class.java) + + const val PATH_VARIABLE_BUCKET = ReferenceArtifactInfo.PATH_VARIABLE_BUCKET + const val PATH_VARIABLE_REF_ID = ReferenceArtifactInfo.PATH_VARIABLE_REF_ID + val VALID_CONTENT_TYPES = arrayOf( + MediaType.APPLICATION_OCTET_STREAM_VALUE, + MediaType.APPLICATION_JSON_VALUE, + MEDIA_TYPE_UNREAL_COMPACT_BINARY, + MEDIA_TYPE_JUPITER_INLINED_PAYLOAD, + MEDIA_TYPE_UNREAL_COMPACT_BINARY_PACKAGE + ) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/BlobNotFoundException.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/BlobNotFoundException.kt new file mode 100644 index 0000000000..382a833538 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/BlobNotFoundException.kt @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.exception + +class BlobNotFoundException(projectId: String, repoName: String, blobId: String) : + RuntimeException("No Blob in Namespace $projectId/$repoName with id $blobId") diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/CbWriterException.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/CbWriterException.kt new file mode 100644 index 0000000000..896de0f73f --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/CbWriterException.kt @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.exception + +class CbWriterException(message: String, e: Exception? = null) : RuntimeException(message, e) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/DdcExceptionHandler.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/DdcExceptionHandler.kt new file mode 100644 index 0000000000..d449bb2a54 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/DdcExceptionHandler.kt @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.exception + +import com.tencent.bkrepo.common.api.constant.MediaTypes +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.service.exception.GlobalExceptionHandler +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.springframework.core.Ordered +import org.springframework.core.annotation.Order +import org.springframework.web.bind.annotation.ExceptionHandler +import org.springframework.web.bind.annotation.RestControllerAdvice + +/** + * ddc服务的响应content-type可能被改为非json，需要单独处理错误响应 + */ +@Order(Ordered.HIGHEST_PRECEDENCE + 1) +@RestControllerAdvice("com.tencent.bkrepo.ddc") +class DdcExceptionHandler(private val exceptionHandler: GlobalExceptionHandler) { + @ExceptionHandler(Exception::class) + fun exception(exception: Exception) { + if (exception is ErrorCodeException) { + exceptionHandler.handleException(exception) + } else { + try { + HttpContextHolder.getResponse().contentType = MediaTypes.APPLICATION_JSON + val m = GlobalExceptionHandler::class.java.getDeclaredMethod("handleException", exception.javaClass) + m.invoke(exceptionHandler, exception) + } catch (e: NoSuchMethodException) { + exceptionHandler.handleException(exception) + } + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/NotImplementedException.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/NotImplementedException.kt new file mode 100644 index 0000000000..cc2724e5b8 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/NotImplementedException.kt @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.exception + +class NotImplementedException(msg: String = "") : RuntimeException(msg) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/ReferenceIsMissingBlobsException.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/ReferenceIsMissingBlobsException.kt new file mode 100644 index 0000000000..13fcfdba5a --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/exception/ReferenceIsMissingBlobsException.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.exception + +import com.tencent.bkrepo.ddc.pojo.ContentHash + +class ReferenceIsMissingBlobsException( + val missingBlobs: List +) : RuntimeException("References is missing these blobs: ${missingBlobs.joinToString(",")}") diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcBlob.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcBlob.kt new file mode 100644 index 0000000000..2fc8f74064 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcBlob.kt @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.model + +import com.tencent.bkrepo.ddc.pojo.ReferenceKey +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document +import java.time.LocalDateTime + +@Document("ddc_blob") +@CompoundIndexes( + CompoundIndex( + name = "projectId_repoName_blobId_idx", + def = "{'projectId': 1, 'repoName': 1, 'blobId': 1}", + unique = true + ), + CompoundIndex( + name = "projectId_repoName_contentId_idx", + def = "{'projectId': 1, 'repoName': 1, 'contentId': 1}", + unique = true + ) +) +data class TDdcBlob( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + + var projectId: String, + var repoName: String, + /** + * blob blake3 hash + */ + var blobId: String, + /** + * 压缩前的blob blake3 hash，如果blob未压缩则contentId与blobId相等 + */ + var contentId: String, + /** + * blob sha256 + */ + var sha256: String, + /** + * blob size + */ + var size: Long, + /** + * 引用了该blob的ref，ref的inline blob和inline blob中引用的blob都会关联到ref + */ + var references: Set? = null +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRef.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRef.kt new file mode 100644 index 0000000000..0985367fec --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRef.kt @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.model + +import org.bson.types.Binary +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document +import java.time.LocalDateTime + +@Document("ddc_ref") +@CompoundIndexes( + CompoundIndex( + name = "projectId_repoName_bucket_key_idx", + def = "{'projectId': 1, 'repoName': 1, 'bucket': 1, 'key': 1}", + unique = true + ) +) +data class TDdcRef( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + var lastAccessDate: LocalDateTime, + + var projectId: String, + var repoName: String, + /** + * ref bucket + */ + var bucket: String, + /** + * ref key, blake3 hash + */ + var key: String, + /** + * 是否所有blob都上传完成 + */ + var finalized: Boolean, + /** + * inline blob id + */ + var blobId: String, + /** + * inline blob，为null时表示inline blob较大，被存放到实际后端存储中 + */ + var inlineBlob: Binary? = null, + /** + * 过期时间 + */ + var expireDate: LocalDateTime? = null, +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Blob.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Blob.kt new file mode 100644 index 0000000000..26a5c61437 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Blob.kt @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +import com.tencent.bkrepo.ddc.artifact.CompressedBlobArtifactInfo +import com.tencent.bkrepo.ddc.model.TDdcBlob +import com.tencent.bkrepo.ddc.utils.DdcUtils.fullPath + +data class Blob( + val projectId: String, + val repoName: String, + val sha256: String, + val fullPath: String, + val size: Long, + val blobId: ContentHash, + val contentId: ContentHash, + val references: Set? = null, +) { + companion object { + fun from(blob: TDdcBlob) = with(blob) { + Blob( + projectId = projectId, + repoName = repoName, + sha256 = sha256, + fullPath = fullPath(), + size = size, + blobId = ContentHash.fromHex(blobId), + contentId = ContentHash.fromHex(contentId), + references = references, + ) + } + + fun from(artifactInfo: CompressedBlobArtifactInfo, sha256: String, size: Long) = with(artifactInfo) { + Blob( + projectId = projectId, + repoName = repoName, + sha256 = sha256, + fullPath = getArtifactFullPath(), + size = size, + blobId = ContentHash.fromHex(compressedContentId!!), + contentId = ContentHash.fromHex(contentId) + ) + } + } +} + +/** + * blob被其他ref或blob引用情况 + * + * bucket和key不为null时候表示被ref引用，blobId不为null时表示被其他blob引用 + */ +data class ReferenceKey( + /** + * ref bucket + */ + val bucket: String? = null, + /** + * ref key + */ + val key: String? = null, + /** + * blob id + */ + val blobId: String? = null, +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/ContentHash.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/ContentHash.kt new file mode 100644 index 0000000000..67b63409ae --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/ContentHash.kt @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +import com.tencent.bkrepo.ddc.utils.BlakeUtils +import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import org.bouncycastle.util.encoders.Hex + +data class ContentHash(val identifier: ByteArray) { + override fun toString(): String = identifier.hex() + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as ContentHash + + return identifier.contentEquals(other.identifier) + } + + override fun hashCode(): Int { + return identifier.contentHashCode() + } + + companion object { + fun fromBlob(blob: ByteArray): ContentHash { + return ContentHash(BlakeUtils.hash(blob)) + } + + fun fromHex(hex: String): ContentHash { + return ContentHash(Hex.decode(hex)) + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/CreateRefResponse.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/CreateRefResponse.kt new file mode 100644 index 0000000000..021971fde7 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/CreateRefResponse.kt @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +data class CreateRefResponse( + val needs: Set +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefId.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefId.kt new file mode 100644 index 0000000000..8b3e4a5714 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefId.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.message.CommonMessageCode + +data class RefId(private var text: String) { + init { + text = text.toLowerCase() + if (text.length != 40) { + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "IoHashKeys must be exactly 40 bytes.") + } + if (text.any { !isValidCharacter(it) }) { + throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "${this.text} contains invalid character.") + } + } + + override fun toString(): String = text + + private fun isValidCharacter(c: Char): Boolean = c in 'a'..'z' || c in '0'..'9' + + companion object { + fun create(s: String): RefId = RefId(s.toLowerCase()) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Reference.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Reference.kt new file mode 100644 index 0000000000..c46c56109c --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Reference.kt @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo +import com.tencent.bkrepo.ddc.model.TDdcRef +import java.time.LocalDateTime + +data class Reference( + val projectId: String, + val repoName: String, + val bucket: String, + val key: RefId, + val finalized: Boolean? = null, + val lastAccessDate: LocalDateTime? = null, + var blobId: ContentHash? = null, + var inlineBlob: ByteArray? = null, +) { + fun fullPath() = "/$bucket/$key" + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Reference + + if (repoName != other.repoName) return false + if (bucket != other.bucket) return false + if (key != other.key) return false + if (lastAccessDate != other.lastAccessDate) return false + if (blobId != other.blobId) return false + if (finalized != other.finalized) return false + if (inlineBlob != null) { + if (other.inlineBlob == null) return false + if (!inlineBlob.contentEquals(other.inlineBlob)) return false + } else if (other.inlineBlob != null) return false + + return true + } + + override fun hashCode(): Int { + var result = repoName.hashCode() + result = 31 * result + bucket.hashCode() + result = 31 * result + key.hashCode() + result = 31 * result + lastAccessDate.hashCode() + result = 31 * result + blobId.hashCode() + result = 31 * result + finalized.hashCode() + result = 31 * result + (inlineBlob?.contentHashCode() ?: 0) + return result + } + + companion object { + fun from(ref: TDdcRef) = Reference( + projectId = ref.projectId, + repoName = ref.repoName, + bucket = ref.bucket, + key = RefId.create(ref.key), + lastAccessDate = ref.lastAccessDate, + blobId = ContentHash.fromHex(ref.blobId), + finalized = ref.finalized, + inlineBlob = ref.inlineBlob?.data + ) + + fun from(artifactInfo: ReferenceArtifactInfo, payload: ByteArray) = with(artifactInfo) { + Reference( + projectId = projectId, + repoName = repoName, + bucket = bucket, + key = refId, + finalized = false, + blobId = ContentHash.fromHex(inlineBlobHash!!), + inlineBlob = payload, + ) + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/UploadCompressedBlobResponse.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/UploadCompressedBlobResponse.kt new file mode 100644 index 0000000000..fa878b1892 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/UploadCompressedBlobResponse.kt @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.pojo + +data class UploadCompressedBlobResponse( + val identifier: String +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/BlobRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/BlobRepository.kt new file mode 100644 index 0000000000..67ece066ea --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/BlobRepository.kt @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.repository + +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.ddc.model.TDdcBlob +import com.tencent.bkrepo.ddc.pojo.ReferenceKey +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.FindAndReplaceOptions +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.inValues +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Repository + +@Repository +class BlobRepository : SimpleMongoDao() { + fun findSmallestByContentId(projectId: String, repoName: String, contentId: String): TDdcBlob? { + val criteria = TDdcBlob::projectId.isEqualTo(projectId) + .and(TDdcBlob::repoName.name).isEqualTo(repoName) + .and(TDdcBlob::contentId.name).isEqualTo(contentId) + val query = Query(criteria).with(Sort.by(Sort.Direction.ASC, TDdcBlob::size.name)) + return findOne(query) + } + + fun findByBlobId(projectId: String, repoName: String, blobId: String): TDdcBlob? { + val criteria = TDdcBlob::projectId.isEqualTo(projectId) + .and(TDdcBlob::repoName.name).isEqualTo(repoName) + .and(TDdcBlob::blobId.name).isEqualTo(blobId) + val query = Query(criteria) + return findOne(query) + } + + fun findByBlobIds(projectId: String, repoName: String, blobIds: Set): List { + val criteria = TDdcBlob::projectId.isEqualTo(projectId) + .and(TDdcBlob::repoName.name).isEqualTo(repoName) + .and(TDdcBlob::blobId.name).inValues(blobIds) + val query = Query(criteria) + return find(query) + } + + fun replace(blob: TDdcBlob): TDdcBlob? { + val criteria = TDdcBlob::projectId.isEqualTo(blob.projectId) + .and(TDdcBlob::repoName.name).isEqualTo(blob.repoName) + .and(TDdcBlob::blobId.name).isEqualTo(blob.blobId) + val options = FindAndReplaceOptions().upsert() + return determineMongoTemplate().findAndReplace(Query(criteria), blob, options) + } + + fun addRefToBlob(projectId: String, repoName: String, bucket: String, refId: String, blobIds: Set) { + val criteria = TDdcBlob::projectId.isEqualTo(projectId) + .and(TDdcBlob::repoName.name).isEqualTo(repoName) + .and(TDdcBlob::blobId.name).inValues(blobIds) + val update = Update().addToSet(TDdcBlob::references.name, ReferenceKey(bucket = bucket, key = refId)) + updateMulti(Query(criteria), update) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefRepository.kt new file mode 100644 index 0000000000..9ab5c7348d --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefRepository.kt @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.repository + +import com.mongodb.DuplicateKeyException +import com.mongodb.client.result.UpdateResult +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.ddc.model.TDdcRef +import org.springframework.data.mongodb.core.FindAndReplaceOptions +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Repository + +@Repository +class RefRepository : SimpleMongoDao() { + fun find(projectId: String, repoName: String, bucket: String, key: String, includePayload: Boolean): TDdcRef? { + val criteria = TDdcRef::projectId.isEqualTo(projectId) + .and(TDdcRef::repoName.name).isEqualTo(repoName) + .and(TDdcRef::bucket.name).isEqualTo(bucket) + .and(TDdcRef::key.name).isEqualTo(key) + val query = Query(criteria) + if (!includePayload) { + query.fields().exclude(TDdcRef::inlineBlob.name) + } + return findOne(query) + } + + fun replace(ref: TDdcRef): TDdcRef? { + val criteria = TDdcRef::projectId.isEqualTo(ref.projectId) + .and(TDdcRef::repoName.name).isEqualTo(ref.repoName) + .and(TDdcRef::bucket.name).isEqualTo(ref.bucket) + .and(TDdcRef::key.name).isEqualTo(ref.key) + val query = Query(criteria) + val options = FindAndReplaceOptions().upsert() + return try { + determineMongoTemplate().findAndReplace(query, ref, options) + } catch (e: DuplicateKeyException) { + findOne(query) + } + } + + fun finalize(projectId: String, repoName: String, bucket: String, key: String): UpdateResult { + val criteria = TDdcRef::projectId.isEqualTo(projectId) + .and(TDdcRef::repoName.name).isEqualTo(repoName) + .and(TDdcRef::bucket.name).isEqualTo(bucket) + .and(TDdcRef::key.name).isEqualTo(key) + val update = Update.update(TDdcRef::finalized.name, true) + return updateFirst(Query(criteria), update) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/Blake3Hash.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/Blake3Hash.kt new file mode 100644 index 0000000000..0eac92ba52 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/Blake3Hash.kt @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import org.bouncycastle.crypto.digests.Blake3Digest +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * 32 bytes blake3 hash + */ +class Blake3Hash( + private val buffer: ByteBuffer +) : Comparable { + + init { + if (buffer.remaining() != NUM_BYTES) { + throw IllegalArgumentException("Blake3Hash must be $NUM_BYTES bytes long") + } + } + + fun getBytes(): ByteBuffer { + return buffer.asReadOnlyBuffer() + } + + override fun compareTo(other: Blake3Hash): Int { + val a = buffer.asReadOnlyBuffer() + val b = other.buffer.asReadOnlyBuffer() + + for (idx in 0 until minOf(a.remaining(), b.remaining())) { + val compare = a.get().compareTo(b.get()) + if (compare != 0) { + return compare + } + } + return a.remaining() - b.remaining() + } + + override fun equals(other: Any?): Boolean { + if (other !is Blake3Hash) return false + return buffer.array().contentEquals(other.buffer.array()) + } + + override fun hashCode(): Int { + return buffer.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN).getInt() + } + + override fun toString(): String { + return buffer.hex() + } + + companion object { + const val NUM_BYTES = 32 + const val NUM_BITS = NUM_BYTES * 8 + val ZERO: Blake3Hash = Blake3Hash(ByteBuffer.allocate(NUM_BYTES)) + fun compute(data: ByteBuffer): Blake3Hash { + val output = ByteArray(32) + val digest = Blake3Digest(NUM_BYTES) + digest.update(data.array(), data.arrayOffset() + data.position(), data.remaining()) + digest.doFinal(output, 0) + return Blake3Hash(ByteBuffer.wrap(output)) + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbArray.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbArray.kt new file mode 100644 index 0000000000..d6b33914a5 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbArray.kt @@ -0,0 +1,104 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import com.tencent.bkrepo.ddc.utils.ByteBufferUtils +import org.bouncycastle.crypto.digests.Blake3Digest +import java.nio.ByteBuffer + +class CbArray(private val innerField: CbField) : Iterable { + val count: Int + get() { + val payload = innerField.payload.asReadOnlyBuffer() + payload.position(VarULong.measure(payload)) + return VarULong.readUnsigned(payload).toInt() + } + + constructor() : this(EMPTY.innerField) + + constructor(data: ByteBuffer, type: CbFieldType = CbFieldType.HasFieldType) : this(CbField(data, type)) + + fun asField(): CbField = innerField + + /** Returns the size of the array in bytes if serialized by itself with no name. */ + fun getSize(): Int = minOf(innerField.getPayloadSize().toInt() + 1, Int.MAX_VALUE) + + /** Calculate the hash of the array if serialized by itself with no name. */ + fun getHash(): Blake3Hash { + val digest = Blake3Digest(Blake3Hash.NUM_BYTES) + + val serializedType = byteArrayOf(innerField.getType().value) + digest.update(serializedType, 0, serializedType.size) + val payload = ByteBuffer.allocate(innerField.payload.remaining()) + payload.put(innerField.payload.asReadOnlyBuffer()) + digest.update(payload.array(), 0, payload.remaining()) + + val result = ByteArray(Blake3Hash.NUM_BYTES) + digest.doFinal(result, 0) + return Blake3Hash(ByteBuffer.wrap(result)) + } + + override fun equals(other: Any?): Boolean { + if (other !is CbArray) return false + return innerField.getType() == other.innerField.getType() && innerField.payload == other.innerField.payload + } + + override fun hashCode(): Int = getHash().getBytes().getInt() + + + /** Copy the array into a buffer of exactly getSize() bytes, with no name. */ + fun copyTo(buffer: ByteBuffer) { + + buffer.put(innerField.getType().value) + buffer.put(innerField.payload.asReadOnlyBuffer()) + } + + fun iterateAttachments(visitor: (CbField) -> Unit) = createIterator().iterateRangeAttachments(visitor) + + /** + * Try to get a view of the array as it would be serialized, such as by CopyTo. + * + * A view is available if the array contains its type and has no name. Access the equivalent + * for other arrays through FCbArray::GetBuffer, FCbArray::Clone, or CopyTo. + */ + fun tryGetView(): Pair = + if (CbFieldUtils.hasFieldName(innerField.typeWithFlags)) { + Pair(false, ByteBufferUtils.EMPTY) + } else { + innerField.tryGetView() + } + + override fun iterator(): Iterator = innerField.iterator() + + fun createIterator(): CbFieldIterator = innerField.createIterator() + + companion object { + val EMPTY: CbArray = CbArray(CbField(ByteBuffer.wrap(byteArrayOf(CbFieldType.Array.value, 1, 0)))) + fun fromFieldNoCheck(field: CbField): CbArray = CbArray(field) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbBinaryAttachment.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbBinaryAttachment.kt new file mode 100644 index 0000000000..ba4851e28f --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbBinaryAttachment.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +/** + * A binary attachment, referenced by [hash] + */ +class CbBinaryAttachment(val hash: IoHash) { + override fun toString(): String = hash.toString() + override fun equals(other: Any?) = other is CbBinaryAttachment && other.hash == hash + override fun hashCode() = hash.a.toInt() + + companion object { + val ZERO: CbBinaryAttachment = CbBinaryAttachment(IoHash.ZERO) + fun fromHash(hash: IoHash): CbBinaryAttachment = CbBinaryAttachment(hash) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbField.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbField.kt new file mode 100644 index 0000000000..34e982e310 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbField.kt @@ -0,0 +1,614 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import com.tencent.bkrepo.ddc.exception.NotImplementedException +import com.tencent.bkrepo.ddc.serialization.CbFieldType.Companion.SIZE_OF_CB_FIELD_TYPE +import com.tencent.bkrepo.ddc.serialization.CbFieldUtils.getType +import com.tencent.bkrepo.ddc.utils.ByteBufferUtils +import org.bouncycastle.crypto.digests.Blake3Digest +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.UUID +import kotlin.experimental.and +import kotlin.experimental.or + + +class CbField : Iterable { + val typeWithFlags: Byte + val fieldData: ByteBuffer + val payload: ByteBuffer + val nameLen: Int + private val payloadOffset: Int + val name: String + + val value: Any? + get() { + val fieldType = getType() + return when (fieldType) { + CbFieldType.None -> NONE + CbFieldType.Null -> null + CbFieldType.Object, CbFieldType.UniformObject -> asObject() + CbFieldType.Array, CbFieldType.UniformArray -> asArray() + CbFieldType.Binary -> asBinary() + CbFieldType.String -> asString() + CbFieldType.IntegerPositive -> asUInt64() + CbFieldType.IntegerNegative -> asInt64() + CbFieldType.Float32 -> asFloat() + CbFieldType.Float64 -> asDouble() + CbFieldType.BoolFalse -> false + CbFieldType.BoolTrue -> true + CbFieldType.ObjectAttachment -> asObjectAttachment() + CbFieldType.BinaryAttachment -> asBinaryAttachment() + CbFieldType.Hash -> asHash() + CbFieldType.Uuid -> asUuid() +// CbFieldType.DateTime -> asDateTime() +// CbFieldType.TimeSpan -> asTimeSpan() + CbFieldType.ObjectId -> asObjectId() + else -> throw NotImplementedException("Unknown field type ($fieldType)") + } + } + var error: CbFieldError = CbFieldError.None + private set + + constructor() : this(ByteBufferUtils.EMPTY, CbFieldType.None) + + constructor(other: CbField) { + typeWithFlags = other.typeWithFlags + fieldData = other.fieldData.asReadOnlyBuffer() + payload = other.payload.asReadOnlyBuffer() + nameLen = other.nameLen + name = other.name + payloadOffset = other.payloadOffset + error = other.error + } + + constructor(data: ByteBuffer, type: CbFieldType = CbFieldType.HasFieldType) : this(data, type.value) + + constructor(data: ByteBuffer, type: Byte) { + var offset = 0 + var fieldTypeValue = type + if (CbFieldUtils.hasFieldType(fieldTypeValue)) { + fieldTypeValue = data.get() or CbFieldType.HasFieldType.value + offset++ + } + if (CbFieldUtils.hasFieldName(fieldTypeValue)) { + nameLen = VarULong.readUnsigned(data).toInt() + val nameArr = ByteArray(nameLen) + for (i in data.position() until data.position() + nameLen) { + nameArr[i - data.position()] = data.get(i) + } + name = String(nameArr) + offset = data.position() + nameLen + } else { + nameLen = 0 + name = "" + } + + typeWithFlags = fieldTypeValue + payloadOffset = offset + + data.position(payloadOffset) + val payloadSize = getPayloadSize(data.slice()) + val limit = minOf(data.limit().toLong(), payloadOffset.toLong() + payloadSize).toInt() + data.limit(limit) + payload = data.slice().asReadOnlyBuffer() + + data.position(0) + fieldData = data.slice().asReadOnlyBuffer() + } + + fun asObject(): CbObject { + return if (CbFieldUtils.isObject(typeWithFlags)) { + error = CbFieldError.None + CbObject.fromFieldNoCheck(this) + } else { + error = CbFieldError.TypeError + CbObject.EMPTY + } + } + + fun asArray(): CbArray { + return if (CbFieldUtils.isArray(typeWithFlags)) { + error = CbFieldError.None + CbArray.fromFieldNoCheck(this) + } else { + error = CbFieldError.TypeError + CbArray.EMPTY + } + } + + fun asBinary(): ByteBuffer { + return asBinary(ByteBufferUtils.EMPTY) + } + + fun asBinary(defaultValue: ByteBuffer): ByteBuffer { + return if (CbFieldUtils.isBinary(typeWithFlags)) { + error = CbFieldError.None + + val buffer = payload.asReadOnlyBuffer() + val length = VarULong.readUnsigned(buffer) + buffer.limit(buffer.position() + length.toInt()) + return buffer.slice() + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + fun asBinaryArray(): ByteArray { + return asBinaryArray(byteArrayOf()) + } + + fun asBinaryArray(defaultValue: ByteArray): ByteArray { + val buffer = asBinary(ByteBuffer.wrap(defaultValue)) + return buffer.array().sliceArray(0 until buffer.remaining()) + } + + fun asString(defaultValue: String = ""): String { + return if (CbFieldUtils.isString(typeWithFlags)) { + val buffer = payload.asReadOnlyBuffer() + val valueSize = VarULong.readUnsigned(buffer) + if (valueSize >= (1L shl 31)) { + error = CbFieldError.RangeError + defaultValue + } else { + error = CbFieldError.None + val arr = ByteBuffer.allocate(valueSize.toInt()) + arr.put(buffer) + String(arr.array()) + } + } else { + error = CbFieldError.TypeError + defaultValue; + } + } + + fun asInt8(defaultValue: Byte = 0): Byte { + return asInteger(defaultValue.toLong(), 7, true).toByte() + } + + fun asInt16(defaultValue: Short = 0): Short { + return asInteger(defaultValue.toLong(), 15, true).toShort() + } + + fun asInt32(defaultValue: Int = 0): Int { + return asInteger(defaultValue.toLong(), 31, true).toInt() + } + + fun asInt64(defaultValue: Long = 0): Long { + return asInteger(defaultValue, 63, true) + } + + fun asUInt8(defaultValue: Byte = 0): Byte { + return asInteger(defaultValue.toLong(), 8, false).toByte() + } + + fun asUInt16(defaultValue: Short = 0): Short { + return asInteger(defaultValue.toLong(), 16, false).toShort() + } + + fun asUInt32(defaultValue: Int = 0): Int { + return asInteger(defaultValue.toLong(), 32, false).toInt() + } + + fun asUInt64(defaultValue: Long = 0): Long { + return asInteger(defaultValue, 64, false) + } + + private fun asInteger(defaultValue: Long, magnitudeBits: Int, isSigned: Boolean): Long { + if (CbFieldUtils.isInteger(typeWithFlags)) { + // 用于判断是否溢出，下面以16位整型为例，几个outOfRangeMask值的示例 + // 读取无符号整型，magnitudeBits为16, outOfRangeMask值为1111_1111 1111_1111 0000_0000 0000_0000 + // 读取有符号整型，magnitudeBits为15, outOfRangeMask值为1111_1111 1111_1111 1000_0000 0000_0000 + val outOfRangeMask = -2L shl (magnitudeBits - 1) + // isNegative为1表示负数，0表示正数 + val isNegative = typeWithFlags and 1.toByte() + + + // isNegative表示正数时，value与magnitude相同， 1111_1110 + // isNegative表示负数时，0000_0001 1111_1111 1111_1111 + val magnitude = VarULong.readUnsigned(payload.asReadOnlyBuffer()) + // TODO 正数是原来的值，负数则直接按位取反，负数的情况下读出来的值与写入的值不一致，应该再加1才是负数的补码表示？ + val value = magnitude xor -isNegative.toLong() + + // 为负数时必须是有符号整型 + return if ((magnitude and outOfRangeMask) == 0L && (isNegative == 0.toByte() || isSigned)) { + error = CbFieldError.None + value + } else { + error = CbFieldError.RangeError + defaultValue + } + } else { + error = CbFieldError.TypeError + return defaultValue + } + } + + /** + * TODO 确认转换结果精度 + */ + fun asFloat(defaultValue: Float = 0.0f): Float { + return when (getType()) { + CbFieldType.IntegerPositive, CbFieldType.IntegerNegative -> { + val isNegative = typeWithFlags and 1.toByte() + val outOfRangeMask: Long = ((1L shl FLT_MANT_DIG) - 1).inv() + + val magnitude = VarULong.readUnsigned(payload.asReadOnlyBuffer()) + isNegative + val isInRange = (magnitude and outOfRangeMask) == 0L + + error = if (isInRange) CbFieldError.None else CbFieldError.RangeError + if (isInRange) { + if (isNegative != 0.toByte()) { + (-magnitude).toFloat() + } else { + magnitude.toFloat() + } + } else { + defaultValue + } + } + + CbFieldType.Float32 -> { + error = CbFieldError.None + payload.asReadOnlyBuffer().float + } + + CbFieldType.Float64 -> { + error = CbFieldError.RangeError + defaultValue + } + + else -> { + error = CbFieldError.TypeError + defaultValue + } + } + } + + fun asDouble(): Double = asDouble(0.0) + + /** + * TODO 确认转换结果精度 + */ + fun asDouble(defaultValue: Double): Double { + when (getType()) { + CbFieldType.IntegerPositive, CbFieldType.IntegerNegative -> { + val isNegative = typeWithFlags and 1.toByte() + val outOfRangeMask = ((1L shl DBL_MANT_DIG) - 1).inv() + + val magnitude = VarULong.readUnsigned(payload.asReadOnlyBuffer()) + isNegative + val isInRange = (magnitude and outOfRangeMask) == 0L + error = if (isInRange) CbFieldError.None else CbFieldError.RangeError + return if (isInRange) { + if (isNegative != 0.toByte()) -magnitude.toDouble() else magnitude.toDouble() + } else { + defaultValue + } + } + + CbFieldType.Float32 -> { + error = CbFieldError.None + return payload.asReadOnlyBuffer().getFloat().toDouble() + } + + CbFieldType.Float64 -> { + error = CbFieldError.None + return payload.asReadOnlyBuffer().getDouble() + } + + else -> { + error = CbFieldError.TypeError + return defaultValue + } + } + } + + fun asBool(): Boolean = asBool(false) + + fun asBool(defaultValue: Boolean): Boolean { + return when (getType()) { + CbFieldType.BoolTrue -> { + error = CbFieldError.None + true + } + + CbFieldType.BoolFalse -> { + error = CbFieldError.None + false + } + + else -> { + error = CbFieldError.TypeError + defaultValue + } + } + } + + fun asObjectAttachment(): CbObjectAttachment = asObjectAttachment(CbObjectAttachment.ZERO) + + fun asObjectAttachment(defaultValue: CbObjectAttachment): CbObjectAttachment { + return if (CbFieldUtils.isObjectAttachment(typeWithFlags)) { + error = CbFieldError.None + CbObjectAttachment(IoHash(payload.asReadOnlyBuffer())) + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + fun asBinaryAttachment(): CbBinaryAttachment = asBinaryAttachment(CbBinaryAttachment.ZERO) + + fun asBinaryAttachment(defaultValue: CbBinaryAttachment): CbBinaryAttachment { + return if (CbFieldUtils.isBinaryAttachment(typeWithFlags)) { + error = CbFieldError.None + CbBinaryAttachment(IoHash(payload.asReadOnlyBuffer())) + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + fun asAttachment(): IoHash { + return asAttachment(IoHash.ZERO) + } + + fun asAttachment(defaultValue: IoHash): IoHash { + return if (CbFieldUtils.isAttachment(typeWithFlags)) { + error = CbFieldError.None + IoHash(payload.asReadOnlyBuffer()) + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + fun asHash(): IoHash { + return asHash(IoHash.ZERO) + } + + fun asHash(defaultValue: IoHash): IoHash { + return if (CbFieldUtils.isHash(typeWithFlags)) { + error = CbFieldError.None + IoHash(payload.asReadOnlyBuffer()) + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + /** + * TODO 与C# 的GUID转换对比 + */ + fun asUuid(defaultValue: UUID = UUID(0, 0)): UUID { + return if (CbFieldUtils.isUuid(typeWithFlags)) { + error = CbFieldError.None + val source = payload.asReadOnlyBuffer() + val target = ByteBuffer.allocate(16) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(source.getInt()) + .putShort(source.getShort()) + .putShort(source.getShort()) + .order(ByteOrder.BIG_ENDIAN) + .putLong(source.getLong()) + target.flip() + return UUID(target.getLong(), target.getLong()) + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + fun asDateTimeTicks(defaultValue: Long = 0L): Long { + return if (CbFieldUtils.isDateTime(typeWithFlags)) { + error = CbFieldError.None + payload.asReadOnlyBuffer().getLong() + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + fun asTimeSpanTicks(defaultValue: Long = 0L): Long { + return if (CbFieldUtils.isTimeSpan(typeWithFlags)) { + error = CbFieldError.None + payload.asReadOnlyBuffer().getLong() + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + fun asObjectId(defaultValue: ByteBuffer = ByteBufferUtils.EMPTY): ByteBuffer { + return if (CbFieldUtils.isObjectId(typeWithFlags)) { + error = CbFieldError.None + payload.asReadOnlyBuffer() + } else { + error = CbFieldError.TypeError + defaultValue + } + } + + fun hasValue() = !CbFieldUtils.isNone(typeWithFlags); + fun hasError() = error != CbFieldError.None; + + /** + * type + name + payload + */ + fun getSize() = SIZE_OF_CB_FIELD_TYPE + getViewNoType().remaining() + + /** + * 获取field hash， 32 bytes + */ + fun getHash(): Blake3Hash { + val digest = Blake3Digest(Blake3Hash.NUM_BYTES) + + val data = ByteArray(1) { CbFieldUtils.getSerializedType(typeWithFlags) } + digest.update(data, 0, data.size) + + val viewNoType = getViewNoType() + val buffer = ByteArray(viewNoType.remaining()) + viewNoType.get(buffer) + digest.update(buffer, 0, buffer.size) + + val hash = ByteArray(Blake3Hash.NUM_BYTES) + digest.doFinal(hash, 0) + + return Blake3Hash(ByteBuffer.wrap(hash)) + } + + override fun hashCode(): Int { + throw NotImplementedException() + } + + override fun equals(other: Any?): Boolean { + return other is CbField && + CbFieldUtils.getSerializedType(typeWithFlags) == CbFieldUtils.getSerializedType(other.typeWithFlags) && + getViewNoType() == other.getViewNoType() + } + + /** + * buffer 大小与[getSize]返回值相同 + */ + fun copyTo(buffer: ByteBuffer) { + buffer.put(CbFieldUtils.getSerializedType(typeWithFlags)) + buffer.put(getViewNoType()) + } + + fun iterateAttachments(visitor: (CbField) -> Unit) { + when (getType()) { + CbFieldType.Object, CbFieldType.UniformObject -> CbObject.fromFieldNoCheck(this).iterateAttachments(visitor) + CbFieldType.Array, CbFieldType.UniformArray -> CbArray.fromFieldNoCheck(this).iterateAttachments(visitor) + CbFieldType.ObjectAttachment, CbFieldType.BinaryAttachment -> visitor(this) + else -> {} + } + } + + fun tryGetView(): Pair { + return if (CbFieldUtils.hasFieldType(typeWithFlags)) { + Pair(true, fieldData.asReadOnlyBuffer()) + } else { + Pair(false, ByteBufferUtils.EMPTY) + } + } + + operator fun get(name: String): CbField = firstOrNull { it.name == name } ?: EMPTY + + fun createIterator(): CbFieldIterator { + val localTypeWithFlags = typeWithFlags + if (CbFieldUtils.hasFields(localTypeWithFlags)) { + val payloadBytes = payload.asReadOnlyBuffer() + val payloadSize = VarULong.readUnsigned(payloadBytes) + + val numByteCount = if (CbFieldUtils.isArray(localTypeWithFlags)) VarULong.measure(payloadBytes) else 0 + if (payloadSize > numByteCount) { + payloadBytes.position(payloadBytes.position() + numByteCount) + var uniformType = CbFieldType.HasFieldType.value + if (CbFieldUtils.hasUniformFields(typeWithFlags)) { + uniformType = payloadBytes.get() + } + return CbFieldIterator(payloadBytes.slice(), uniformType) + } + } + return CbFieldIterator(ByteBufferUtils.EMPTY, CbFieldType.HasFieldType.value) + } + + override fun iterator(): Iterator { + return createIterator() + } + + private fun getViewNoType(): ByteBuffer { + val nameSize = if (CbFieldUtils.hasFieldName(typeWithFlags)) { + nameLen + VarULong.measureUnsigned(nameLen.toLong()) + } else { + 0 + } + + val buffer = fieldData.asReadOnlyBuffer() + buffer.position(payloadOffset - nameSize) + return buffer.slice() + } + + fun getType(): CbFieldType = getType(typeWithFlags) + + fun getPayloadSize(payload: ByteBuffer = this.payload): Long { + return when (getType()) { + CbFieldType.None, CbFieldType.Null -> 0L + CbFieldType.Object, + CbFieldType.UniformObject, + CbFieldType.Array, + CbFieldType.UniformArray, + CbFieldType.Binary, + CbFieldType.String -> { + val buffer = payload.asReadOnlyBuffer() + val payloadSize = VarULong.readUnsigned(buffer) + buffer.position() + payloadSize + } + + CbFieldType.IntegerPositive, CbFieldType.IntegerNegative -> { + VarULong.measure(payload).toLong() + } + + CbFieldType.Float32 -> 4L + CbFieldType.Float64 -> 8L + CbFieldType.BoolFalse, CbFieldType.BoolTrue -> 0L + CbFieldType.ObjectAttachment, CbFieldType.BinaryAttachment, CbFieldType.Hash -> 20L + CbFieldType.Uuid -> 16L + CbFieldType.DateTime, CbFieldType.TimeSpan -> 8L + CbFieldType.ObjectId -> 12L + else -> 0L + } + } + + enum class CbFieldError(val value: Byte) { + /** + * 未出错 + */ + None(0x01), + + /** + * 类型错误 + */ + TypeError(0x02), + + /** + * 请求数据范围错误 + */ + RangeError(0x03), + } + + class NoneValueType + + companion object { + private const val FLT_MANT_DIG = 24 + private const val DBL_MANT_DIG = 53 + val NONE: NoneValueType = NoneValueType() + val EMPTY: CbField = CbField() + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldIterator.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldIterator.kt new file mode 100644 index 0000000000..d7cd51b9df --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldIterator.kt @@ -0,0 +1,118 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import com.tencent.bkrepo.ddc.exception.NotImplementedException +import com.tencent.bkrepo.ddc.utils.ByteBufferUtils +import java.nio.ByteBuffer + +class CbFieldIterator : Iterator { + + private var nextData: ByteBuffer + private val uniformType: Byte + var current: CbField = CbField.EMPTY + private set + + constructor() : this(ByteBufferUtils.EMPTY, CbFieldType.HasFieldType.value) + + private constructor(field: CbField) { + nextData = ByteBufferUtils.EMPTY + current = field + uniformType = CbFieldType.HasFieldType.value + } + + constructor(data: ByteBuffer, uniformType: Byte) { + this.nextData = data.asReadOnlyBuffer() + this.uniformType = uniformType + moveNext() + } + + constructor(other: CbFieldIterator) { + nextData = other.nextData.asReadOnlyBuffer() + uniformType = other.uniformType + current = other.current + } + + fun isValid(): Boolean { + return current.getType() != CbFieldType.None + } + + fun iterateRangeAttachments(visitor: (CbField) -> Unit) { + if (CbFieldUtils.hasFieldType(current.typeWithFlags)) { + for (it in CbFieldIterator(this)) { + if (CbFieldUtils.mayContainAttachments(it.typeWithFlags)) { + it.iterateAttachments(visitor) + } + } + } else if (CbFieldUtils.mayContainAttachments(current.typeWithFlags)) { + for (it in CbFieldIterator(this)) { + it.iterateAttachments(visitor) + } + } + } + + fun moveNext(): Boolean { + return if (nextData.remaining() > 0) { + current = CbField(nextData.asReadOnlyBuffer(), uniformType) + nextData.position(current.fieldData.remaining()) + nextData = nextData.slice() + true + } else { + current = CbField.EMPTY + false + } + } + + override fun equals(other: Any?): Boolean { + throw NotImplementedException() + } + + override fun hashCode(): Int { + throw NotImplementedException() + } + + override fun hasNext(): Boolean { + return isValid() + } + + override fun next(): CbField { + val field = current + moveNext() + return field + } + + companion object { + fun makeSingle(field: CbField): CbFieldIterator { + return CbFieldIterator(field) + } + + fun makeRange(view: ByteBuffer, type: Byte = CbFieldType.HasFieldType.value): CbFieldIterator { + return CbFieldIterator(view, type) + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldType.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldType.kt new file mode 100644 index 0000000000..b1ef4f3288 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldType.kt @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +/** + * 通过CompactBinary序列化后的字段类型 + */ +enum class CbFieldType(val value: Byte) { + None(0x00), + Null(0x01), + + /** + * 不带类型的Object，子字段需要写入类型信息 + */ + Object(0x02), + + /** + * 带类型的Object，所有子字段使用同一种类型 + */ + UniformObject(0x03), + + /** + * 不带类型的数组，子字段需要写入类型信息 + */ + Array(0x04), + + /** + * 带类型的Array，所有子字段使用同一种类型 + */ + UniformArray(0x05), + Binary(0x06), + String(0x07), + IntegerPositive(0x08), + IntegerNegative(0x09), + Float32(0x0a), + Float64(0x0b), + BoolFalse(0x0c), + BoolTrue(0x0d), + ObjectAttachment(0x0e), + + /** + * BinaryAttachment is a reference to a binary attachment stored externally. + * + * Payload is a 160-bit hash digest of the referenced binary data. + */ + BinaryAttachment(0x0f), + + /** + * Hash. Payload is a 160-bit hash digest. + */ + Hash(0x10), + Uuid(0x11), + /** + * Date and time between 0001-01-01 00:00:00.0000000 and 9999-12-31 23:59:59.9999999. + * + * Payload is a big endian int64 count of 100ns ticks since 0001-01-01 00:00:00.0000000. + */ + DateTime(0x12), + TimeSpan(0x13), + ObjectId(0x14), + CustomById(0x1e), + CustomByName(0x1f), + HasFieldType(0x40), + HasFieldName(0x80.toByte()); + + companion object { + const val SIZE_OF_CB_FIELD_TYPE = 1 + fun getByValue(value: Byte): CbFieldType? { + return if (value >= None.value && value <= Object.value) { + CbFieldType.values()[value.toInt()] + } else { + CbFieldType.values().firstOrNull { it.value == value } + } + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldUtils.kt new file mode 100644 index 0000000000..0da57c4046 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldUtils.kt @@ -0,0 +1,151 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import kotlin.experimental.and + +object CbFieldUtils { + private const val SERIALIZED_TYPE_MASK: Byte = 0b1001_1111.toByte() + private const val TYPE_MASK: Byte = 0b0001_1111.toByte() + + private const val OBJECT_MASK: Byte = 0b0001_1110.toByte() + private const val OBJECT_BASE: Byte = 0b0000_0010.toByte() + + private const val ARRAY_MASK: Byte = 0b0001_1110.toByte() + private const val ARRAY_BASE: Byte = 0b0000_0100.toByte() + + private const val INTEGER_MASK: Byte = 0b0011_1110.toByte() + private const val INTEGER_BASE: Byte = 0b0000_1000.toByte() + + private const val FLOAT_MASK: Byte = 0b0001_1100.toByte() + private const val FLOAT_BASE: Byte = 0b0000_1000.toByte() + + private const val BOOL_MASK: Byte = 0b0001_1110.toByte() + private const val BOOL_BASE: Byte = 0b0000_1100.toByte() + + private const val ATTACHMENT_MASK: Byte = 0b0001_1110.toByte() + private const val ATTACHMENT_BASE: Byte = 0b0000_1110.toByte() + + fun getType(typeWithFlags: Byte): CbFieldType { + return CbFieldType.getByValue(typeWithFlags and TYPE_MASK)!! + } + + fun getSerializedType(typeWithFlags: Byte): Byte { + return typeWithFlags and SERIALIZED_TYPE_MASK + } + + fun hasFieldType(type: Byte): Boolean { + return (type and CbFieldType.HasFieldType.value) != 0.toByte() + } + + fun hasFieldName(type: Byte): Boolean { + return (type and CbFieldType.HasFieldName.value) != 0.toByte() + } + + fun isNone(type: Byte): Boolean { + return getType(type) == CbFieldType.None + } + + fun isNull(type: Byte): Boolean { + return getType(type) == CbFieldType.Null + } + + fun isObject(type: Byte): Boolean { + return (type and OBJECT_MASK) == OBJECT_BASE + } + + fun isArray(type: Byte): Boolean { + return (type and ARRAY_MASK) == ARRAY_BASE + } + + fun isBinary(type: Byte): Boolean { + return getType(type) == CbFieldType.Binary + } + + fun isString(type: Byte): Boolean { + return getType(type) == CbFieldType.String + } + + fun isInteger(type: Byte): Boolean { + return (type and INTEGER_MASK) == INTEGER_BASE + } + + fun isFloat(type: Byte): Boolean { + return (type and FLOAT_MASK) == FLOAT_BASE + } + + fun isBool(type: Byte): Boolean { + return (type and BOOL_MASK) == BOOL_BASE + } + + fun isObjectAttachment(type: Byte): Boolean { + return getType(type) == CbFieldType.ObjectAttachment + } + + fun isBinaryAttachment(type: Byte): Boolean { + return getType(type) == CbFieldType.BinaryAttachment + } + + fun isAttachment(type: Byte): Boolean { + return (type and ATTACHMENT_MASK) == ATTACHMENT_BASE + } + + fun isHash(type: Byte): Boolean { + return getType(type) === CbFieldType.Hash || isAttachment(type) + } + + fun isUuid(type: Byte): Boolean { + return getType(type) === CbFieldType.Uuid + } + + fun isDateTime(type: Byte): Boolean { + return getType(type) === CbFieldType.DateTime + } + + fun isTimeSpan(type: Byte): Boolean { + return getType(type) === CbFieldType.TimeSpan + } + + fun isObjectId(type: Byte): Boolean { + return getType(type) === CbFieldType.ObjectId + } + + fun hasFields(type: Byte): Boolean { + val noFlags: CbFieldType = getType(type) + return noFlags >= CbFieldType.Object && noFlags <= CbFieldType.UniformArray + } + + fun hasUniformFields(type: Byte): Boolean { + val localType: CbFieldType = getType(type) + return localType === CbFieldType.UniformObject || localType === CbFieldType.UniformArray + } + + fun mayContainAttachments(type: Byte): Boolean { + return isObject(type) || isArray(type) || isAttachment(type) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObject.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObject.kt new file mode 100644 index 0000000000..d947c21fe4 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObject.kt @@ -0,0 +1,202 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization; + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.io.SegmentedStringWriter +import com.fasterxml.jackson.databind.ObjectMapper +import com.tencent.bkrepo.ddc.exception.NotImplementedException +import com.tencent.bkrepo.ddc.serialization.CbFieldType.Companion.SIZE_OF_CB_FIELD_TYPE +import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import com.tencent.bkrepo.ddc.utils.ByteBufferUtils +import org.bouncycastle.crypto.digests.Blake3Digest +import java.nio.ByteBuffer + +class CbObject : Iterable { + + val innerField: CbField + + constructor(field: CbField) { + innerField = CbField(field.fieldData.asReadOnlyBuffer(), field.typeWithFlags) + } + + constructor(buffer: ByteBuffer, fieldType: CbFieldType = CbFieldType.HasFieldType) { + innerField = CbField(buffer.asReadOnlyBuffer(), fieldType) + } + + fun find(name: String): CbField { + return innerField[name] + } + + fun findIgnoreCase(name: String): CbField { + return innerField.firstOrNull { it.name.equals(name, ignoreCase = true) } ?: CbField.EMPTY + } + + operator fun get(name: String): CbField = innerField[name] + + fun asField() = innerField + + /** + * Returns the size of the object in bytes if serialized by itself with no name. + */ + fun getSize(): Int { + return SIZE_OF_CB_FIELD_TYPE + innerField.payload.remaining() + } + + /** + * Calculate the hash of the object if serialized by itself with no name. + */ + fun getHash(): Blake3Hash { + val digest = Blake3Digest() + + val temp = byteArrayOf(innerField.getType().value) + digest.update(temp, 0, temp.size) + val payload = ByteBuffer.allocate(innerField.payload.remaining()) + payload.put(innerField.payload.asReadOnlyBuffer()) + digest.update(payload.array(), 0, payload.remaining()) + + val data = ByteArray(Blake3Hash.NUM_BYTES) + digest.doFinal(data, 0) + return Blake3Hash(ByteBuffer.wrap(data)) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is CbObject) return false + return innerField.getType() == other.innerField.getType() && innerField.payload == other.innerField.payload + } + + override fun hashCode(): Int { + return getHash().getBytes().getInt() + } + + fun copyTo(buffer: ByteBuffer) { + buffer.put(innerField.getType().value) + buffer.put(innerField.payload.asReadOnlyBuffer()) + } + + fun iterateAttachments(visitor: (CbField) -> Unit) { + createIterator().iterateRangeAttachments(visitor) + } + + fun getView(): ByteBuffer { + val (result, view) = tryGetView() + if (!result) { + val data = ByteBuffer.allocate(getSize()) + copyTo(data) + return data + } + return view + } + + fun tryGetView(): Pair { + if (CbFieldUtils.hasFieldName(innerField.typeWithFlags)) { + return Pair(false, ByteBufferUtils.EMPTY) + } + return innerField.tryGetView() + } + + fun createIterator() = innerField.createIterator() + + override fun iterator(): Iterator { + return innerField.iterator() + } + + fun toJson(mapper: ObjectMapper): String { + val factory = mapper.factory + val sw = SegmentedStringWriter(factory._getBufferRecycler()) + mapper.factory.createGenerator(sw).use { g -> + g.writeStartObject() + for (field in innerField) { + writeField(field, g) + } + g.writeEndObject() + } + + return sw.getAndClear() + } + + private fun writeField(field: CbField, writer: JsonGenerator) { + if (CbFieldUtils.isObject(field.typeWithFlags)) { + if (field.nameLen != 0) { + writer.writeObjectFieldStart(field.name) + } else { + writer.writeStartObject() + } + val obj = field.asObject() + for (objField in obj.innerField) { + writeField(objField, writer) + } + writer.writeEndObject() + } else if (CbFieldUtils.isArray(field.typeWithFlags)) { + writer.writeArrayFieldStart(field.name) + val array = field.asArray() + for (objectField in array) { + writeField(objectField, writer) + } + writer.writeEndArray() + } else if (CbFieldUtils.isInteger(field.typeWithFlags)) { + if (field.getType() == CbFieldType.IntegerNegative) { + writer.writeNumberField(field.name, -field.asInt64()) + } else { + writer.writeNumberField(field.name, field.asUInt64()) + } + } else if (CbFieldUtils.isBool(field.typeWithFlags)) { + writer.writeBooleanField(field.name, field.asBool()) + } else if (CbFieldUtils.isNull(field.typeWithFlags)) { + writer.writeNull() + } else if (CbFieldUtils.isDateTime(field.typeWithFlags)) { + throw NotImplementedException() +// writer.writeStringField(field.name, field.asDateTime()) + } else if (CbFieldUtils.isHash(field.typeWithFlags)) { + writer.writeStringField(field.name, field.asHash().toString()) + } else if (CbFieldUtils.isString(field.typeWithFlags)) { + writer.writeStringField(field.name, field.asString()) + } else if (CbFieldUtils.isObjectId(field.typeWithFlags)) { + writer.writeStringField(field.name, field.asObjectId().hex()) + } else { + throw NotImplementedException("Unhandled type ${field.getType()} when attempting to convert to json") + } + } + + companion object { + val EMPTY = fromFieldNoCheck(CbField(ByteBuffer.wrap(byteArrayOf(CbFieldType.Object.value, 0)))) + + fun fromFieldNoCheck(field: CbField): CbObject { + return CbObject(field) + } + + fun build(build: (writer: CbWriter) -> Unit): CbObject { + val writer = CbWriter() + writer.beginObject() + build(writer) + writer.endObject() + return CbObject(ByteBuffer.wrap(writer.toByteArray())) + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObjectAttachment.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObjectAttachment.kt new file mode 100644 index 0000000000..49eeb5c2f0 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbObjectAttachment.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +/** + * An object attachment, referenced by [hash] + */ +class CbObjectAttachment(val hash: IoHash) { + override fun toString(): String = hash.toString() + override fun equals(other: Any?) = other is CbObjectAttachment && other.hash == hash + override fun hashCode() = hash.a.toInt() + + companion object { + val ZERO: CbObjectAttachment = CbObjectAttachment(IoHash.ZERO) + fun fromHash(hash: IoHash): CbObjectAttachment = CbObjectAttachment(hash) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriter.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriter.kt new file mode 100644 index 0000000000..f28afa7663 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriter.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import java.nio.ByteBuffer + +class CbWriter : CbWriterBase() { + override fun allocateChunk(minSize: Int): ByteBuffer { + return ByteBuffer.allocate(maxOf(minSize, DEFAULT_CHUNK_SIZE)) + } + + companion object { + const val DEFAULT_CHUNK_SIZE: Int = 1024 + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterBase.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterBase.kt new file mode 100644 index 0000000000..1e5d87c0bf --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterBase.kt @@ -0,0 +1,332 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import com.tencent.bkrepo.ddc.exception.CbWriterException +import com.tencent.bkrepo.ddc.serialization.CbFieldType.Companion.SIZE_OF_CB_FIELD_TYPE +import com.tencent.bkrepo.ddc.utils.BlakeUtils +import com.tencent.bkrepo.ddc.utils.ByteBufferUtils +import java.nio.ByteBuffer +import java.util.ArrayDeque +import kotlin.experimental.or + + +abstract class CbWriterBase : ICbWriter { + + private val rootScope = Scope(fieldType = CbFieldType.Array) + + /** + * 栈顶元素为当前正在操作的scope + */ + private val openScopes = ArrayDeque() + private val freeScopes = ArrayDeque() + + private var buffer = ByteBufferUtils.EMPTY + + /** + * 当前正在操作的字段buffer起始位置，inclusive + */ + private var bufferPos = 0 + + /** + * 当前正在操作的字段buffer结束位置，exclusive + */ + private var bufferEnd = 0 + + init { + openScopes.push(rootScope) + } + + protected fun reset() { + addChildrenToFreeList(rootScope) + rootScope.reset() + } + + private fun addChildrenToFreeList(root: Scope) { + var child = root.firstChild + while (child != null) { + addChildrenToFreeList(child) + child.reset() + freeScopes.push(child) + child = child.nextSibling + } + } + + private fun allocScope(): Scope { + return if (freeScopes.isNotEmpty()) { + freeScopes.pop() + } else { + Scope() + } + } + + /** + * 添加子字段数据 + */ + private fun addLeafData(data: ByteBuffer) { + val scope = allocScope() + scope.data = data.asReadOnlyBuffer() + scope.length = data.remaining() + + val currentScope = openScopes.peek() + currentScope.addChild(scope) + } + + /** + * 创建新字段 + */ + private fun enterScope(fieldType: CbFieldType, name: String?): Scope { + val currentScope = openScopes.peek() + + val scope = allocScope() + scope.fieldType = fieldType + scope.writeFieldType = currentScope.uniformFieldType == CbFieldType.None + scope.name = name + + currentScope.addChild(scope) + + openScopes.push(scope) + return scope + } + + /** + * 结束字段创建，写入header数据到scope的data字段 + */ + private fun leaveScope() { + writeFields() + val scope = openScopes.peek() + val childrenLength = scope.childrenLength() + + if (scope.fieldType != CbFieldType.None) { + val typeAndNameLength = scope.typeAndNameLength() + val payloadTypeAndItemCountLength = scope.payloadTypeAndItemCountLength() + val payloadLength = payloadTypeAndItemCountLength.toLong() + childrenLength.toLong() + val payloadLengthBytes = VarULong.measureUnsigned(payloadLength) + + val header = allocate(typeAndNameLength + payloadLengthBytes + payloadTypeAndItemCountLength) + scope.data = header.asReadOnlyBuffer() + bufferPos = bufferEnd + + if (scope.writeFieldType) { + if (scope.name.isNullOrEmpty()) { + header.put(scope.fieldType.value) + } else { + header.put(scope.fieldType.value or CbFieldType.HasFieldName.value) + VarULong.writeUnsigned(header, scope.name!!.length.toLong()) + header.put(scope.name!!.toByteArray()) + } + } + VarULong.writeUnsigned(header, payloadLength) + + if (CbFieldUtils.isArray(scope.fieldType.value)) { + VarULong.writeUnsigned(header, scope.itemCount.toLong()) + } + + if (scope.fieldType === CbFieldType.UniformObject || scope.fieldType === CbFieldType.UniformArray) { + header.put(scope.uniformFieldType.value) + } + } + + scope.length = childrenLength + scope.data!!.remaining() + openScopes.pop() + } + + override fun beginObject(name: String?) { + writeFields() + + val parentScope = openScopes.peek() + parentScope.itemCount++ + + enterScope(CbFieldType.Object, name) + } + + override fun endObject() = leaveScope() + + override fun beginArray(name: String?, elementType: CbFieldType) { + writeFields() + + val parentScope = openScopes.peek() + parentScope.itemCount++ + + val fieldType = if (elementType == CbFieldType.None) { + CbFieldType.Array + } else { + CbFieldType.UniformArray + } + + val scope = enterScope(fieldType, name) + scope.uniformFieldType = elementType + } + + override fun endArray() = leaveScope() + + private fun writeFields() { + if (bufferPos < bufferEnd) { + buffer.position(bufferPos) + buffer.limit(bufferEnd) + addLeafData(buffer.slice()) + bufferPos = bufferEnd + } + } + + private fun allocate(length: Int): ByteBuffer { + if (bufferEnd + length > buffer.capacity()) { + writeFields() + buffer = allocateChunk(length) + bufferPos = 0 + bufferEnd = 0 + } + + buffer.position(bufferEnd) + buffer.limit(bufferEnd + length) + val data = buffer.slice() + bufferEnd += length + + return data + } + + protected abstract fun allocateChunk(minSize: Int): ByteBuffer + + override fun writeField(type: CbFieldType, name: String?, length: Int): ByteBuffer { + writeFieldHeader(type, name) + val buffer = allocate(length) + if (openScopes.count() == 1) { + // TODO 确认是否需要改成所有情况都将数据刷到scope child + writeFields() + } + return buffer + } + + private fun writeFieldHeader(type: CbFieldType, name: String? = null) { + val scope = openScopes.peek() + if (name.isNullOrEmpty()) { + val scopeType = scope.fieldType + if (!CbFieldUtils.isArray(scopeType.value)) { + throw CbWriterException("Anonymous fields are not allowed within fields of type $scopeType") + } + + val elementType = scope.uniformFieldType + if (elementType == CbFieldType.None) { + allocate(1).put(type.value) + } else if (elementType != type) { + throw CbWriterException("Mismatched type for uniform array - expected $elementType, not $type") + } + + scope.itemCount++ + } else { + val scopeType = scope.fieldType + if (!CbFieldUtils.isObject(scopeType.value)) { + throw CbWriterException("Named fields are not allowed within fields of type $scopeType") + } + + val elementType = scope.uniformFieldType + + val nameVarIntLength = VarULong.measureUnsigned(name.length.toLong()) + if (elementType == CbFieldType.None) { + val buffer = allocate(SIZE_OF_CB_FIELD_TYPE + nameVarIntLength + name.length) + buffer.put(type.value or CbFieldType.HasFieldName.value) + writeBinaryPayload(buffer, name.toByteArray()) + } else { + if (elementType != type) { + throw CbWriterException("Mismatched type for uniform object - expected $elementType, not $type") + } + val buffer = allocate(name.length) + writeBinaryPayload(buffer, name.toByteArray()) + } + + scope.itemCount++ + } + } + + override fun writeReference(data: ByteBuffer) { + writeFields() + addLeafData(data) + } + + private fun writeBinaryPayload(output: ByteBuffer, value: ByteArray) { + VarULong.writeUnsigned(output, value.size.toLong()) + output.put(value) + } + + fun getSize(): Int { + if (openScopes.count() > 1) { + throw CbWriterException("Unfinished scope in writer") + } + + var length = 0 + var child = rootScope.firstChild + while (child != null) { + length += child.length + child = child.nextSibling + } + + return length + } + + fun copyTo(buffer: ByteBuffer) { + copy(rootScope, buffer) + } + + private fun copy(scope: Scope, buffer: ByteBuffer) { + if (scope.dataSize() > 0) { + buffer.put(scope.data) + } + var child = scope.firstChild + while (child != null) { + copy(child, buffer) + child = child.nextSibling + } + } + + fun computeHash(): ByteArray { + return BlakeUtils.hash(getSegments()) + } + + fun toByteArray(): ByteArray { + val buffer = ByteBuffer.allocate(getSize()) + copyTo(buffer) + return buffer.array() + } + + fun getSegments(): List { + val segments = mutableListOf() + getSegments(rootScope, segments) + return segments + } + + private fun getSegments(scope: Scope, segments: MutableList) { + if (scope.dataSize() > 0) { + segments.add(scope.data!!) + } + var child = scope.firstChild + while (child != null) { + getSegments(child, segments) + child = child.nextSibling + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/ICbWriter.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/ICbWriter.kt new file mode 100644 index 0000000000..51204b012b --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/ICbWriter.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import java.nio.ByteBuffer + +interface ICbWriter { + fun beginObject(name: String? = null) + + fun endObject() + + fun beginArray(name: String? = null, elementType: CbFieldType = CbFieldType.None) + + fun endArray() + + fun writeField(type: CbFieldType, name: String? = null, length: Int): ByteBuffer + + fun writeReference(data: ByteBuffer) +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/IoHash.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/IoHash.kt new file mode 100644 index 0000000000..0ee5c446c5 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/IoHash.kt @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import java.nio.ByteBuffer +import kotlin.math.sign + +/** + * 20 bytes Blake3 hash + */ +data class IoHash( + val a: Long, + val b: Long, + val c: Int +) : Comparable { + companion object { + const val NUM_BYTES = 20 + const val NUM_BITS = NUM_BYTES * 8 + val ZERO = IoHash(0L, 0L, 0) + } + + constructor(bytes: ByteBuffer) : this(bytes.getLong(), bytes.getLong(), bytes.getInt()) + + fun toByteArray(): ByteArray { + val arr = ByteArray(NUM_BYTES) + val buffer = ByteBuffer.wrap(arr) + copyTo(buffer) + return arr + } + + fun copyTo(buffer: ByteBuffer) { + buffer.putLong(a) + buffer.putLong(b) + buffer.putInt(c) + } + + override fun compareTo(other: IoHash): Int { + return when { + a != other.a -> (a - other.a).sign + b != other.b -> (b - other.b).sign + else -> (c - other.c).sign + } + } + + override fun toString(): String { + val buffer = ByteBuffer.allocate(NUM_BYTES) + copyTo(buffer) + buffer.flip() + return buffer.hex() + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is IoHash) return false + return a == other.a && b == other.b && c == other.c + } + + override fun hashCode(): Int { + return a.toInt() + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/Scope.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/Scope.kt new file mode 100644 index 0000000000..5950df2c65 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/Scope.kt @@ -0,0 +1,145 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import java.nio.ByteBuffer + +/** + * CompactBinary的字段 + */ +data class Scope( + /** + * 字段类型 + */ + var fieldType: CbFieldType = CbFieldType.None, + + /** + * 序列化时是否写入字段类型 + * [CbFieldType.UniformArray]、[CbFieldType.UniformObject]统一设置了子字段的类型，子字段[writeFieldType]的值就是false + */ + var writeFieldType: Boolean = true, + + /** + * 子字段类型 + */ + var uniformFieldType: CbFieldType = CbFieldType.None, + + /** + * 字段名 + */ + var name: String? = null, + /** + * 子字段数量 + */ + var itemCount: Int = 0, + /** + * 当前字段的数据 + * Array或Object存的是header（比如字段类型、字段名）的数据 + * 其他类型存的是实际数据 + */ + var data: ByteBuffer? = null, + /** + * [data]的长度加上所有子字段的[data]长度 + */ + var length: Int = 0, + /** + * 第一个子字段 + */ + var firstChild: Scope? = null, + /** + * 最后一个子字段 + */ + var lastChild: Scope? = null, + /** + * 子字段是链表结构，通过[nextSibling]可以找到相邻字段 + */ + var nextSibling: Scope? = null, +) { + public fun reset() + { + fieldType = CbFieldType.None + writeFieldType = true + uniformFieldType = CbFieldType.None + name = null + itemCount = 0 + data = null + length = 0 + firstChild = null + lastChild = null + nextSibling = null + } + + fun addChild(child: Scope) { + if (lastChild == null) { + firstChild = child + } else { + lastChild!!.nextSibling = child + } + lastChild = child + } + + /** + * 获取所有子字段长度总和 + */ + fun childrenLength(): Int { + var childrenLength = 0 + var child = firstChild + while (child != null) { + childrenLength += child.length + child = child.nextSibling + } + return childrenLength + } + + /** + * 获取字段序列化后类型与字段名信息的长度 + */ + fun typeAndNameLength(): Int { + var length = 0 + if (writeFieldType) { + length++ + if (!name.isNullOrEmpty()) { + length += VarULong.measureUnsigned(name!!.length.toLong()) + name!!.length + } + } + return length + } + + fun payloadTypeAndItemCountLength(): Int { + var length = 0 + if (CbFieldUtils.isArray(fieldType.value)) { + length += VarULong.measureUnsigned(itemCount.toLong()) + } + if (fieldType.value == CbFieldType.UniformObject.value || fieldType.value == CbFieldType.UniformArray.value) { + length++ + } + return length + } + + fun dataSize() = data?.remaining() ?: 0 +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/VarULong.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/VarULong.kt new file mode 100644 index 0000000000..3415538e45 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/serialization/VarULong.kt @@ -0,0 +1,149 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import java.nio.ByteBuffer +import kotlin.experimental.and +import kotlin.experimental.inv + + +/** + * 可变长度Long，具体表示规则参考下方示例 + * + * 小于等于127的整形只需要一个字节表示, 最大值为0x01111111 + * 大于127的整形需要多个字节表示,第一个字节每一位表示除第一个字节外还需要的字节，例如 + * 129 二进制为0x10000000 0x10000001 + * 65537 二进制为 0x11000001 0x00000000 0x00000001 + * 大于等于2^56需要使用9个字节表示 + * 2^56+1的二进制为 + * 0x11111111 0x00000001 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000 0x00000001 + */ +object VarULong { + /** + * 判断buffer中的数字需要多少字节进行编码 + * + * @return 需要的字节数，取值范围[1,9] + */ + fun measure(buffer: ByteBuffer): Int { + val b: Byte = buffer.get(buffer.position()) + val i = b.toInt() + if (i in 0..127) { + return 1 + } + return Integer.numberOfLeadingZeros(b.inv().toInt()) - 23 + } + + /** + * 测算需要多少字节才能表示指定的无符号Long + * + * @param unsignedValue 待测算的无符号整形Long + * + * @return 所需字节数 + */ + fun measureUnsigned(unsignedValue: Long): Int { + var count = 0 + var v = unsignedValue + do { + v = v ushr 7 + count++ + + } while (v != 0L) + return minOf(count, 9) + } + + /** + * 写入一个变长Long到buffer中 + * + * @param buffer 待写入的buffer + * @param unsignedValue 待写入的值 + * + * @return 写入的字节数 + */ + fun writeUnsigned(buffer: ByteBuffer, unsignedValue: Long): Int { + var v = unsignedValue + val position = buffer.position() + val byteCount = measureUnsigned(v) + for (idx in 1 until byteCount) { + buffer.put(position + byteCount - idx, v.toByte()) + v = v ushr 8 + } + val firstByte = ((0xff shl (9 - byteCount)) or v.toInt()).toByte() + buffer.put(position, firstByte) + buffer.position(position + byteCount) + return byteCount + } + + /** + * 从buffer中读取变长Long + * + * @param buffer 待读取的buffer，会改变buffer position + * + * @return buffer中存储的值 + */ + fun readUnsigned(buffer: ByteBuffer): Long { + val numBytes = measure(buffer) + var value = (buffer.get() and (0xff ushr numBytes).toByte()).toLong() + for (i in 1 until numBytes) { + value = value shl 8 + value = value or (buffer.get().toLong() and 0xff) + } + return value + } + + /** + * 写入有符号数 + * + * @param buffer 待写入的buffer + * @param signedValue 待写入的值 + * + * @return 写入的字节数 + */ + fun writeSigned(buffer: ByteBuffer, signedValue: Long): Int { + return writeUnsigned(buffer, encodeSigned(signedValue)) + } + + /** + * 从buffer中读取变长Long + * + * @param buffer 待读取的buffer，会改变buffer position + * + * @return buffer中存储的值 + */ + fun readSigned(buffer: ByteBuffer): Long { + val value = readUnsigned(buffer) + return decodeSigned(value) + } + + private fun encodeSigned(value: Long): Long { + return (value shr 63) xor (value shl 1) + } + + private fun decodeSigned(value: Long): Long { + return -(value and 1) xor (value ushr 1) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt new file mode 100644 index 0000000000..9b3da23096 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt @@ -0,0 +1,110 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.service + +import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream +import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.ddc.exception.BlobNotFoundException +import com.tencent.bkrepo.ddc.model.TDdcBlob +import com.tencent.bkrepo.ddc.pojo.Blob +import com.tencent.bkrepo.ddc.pojo.Reference +import com.tencent.bkrepo.ddc.repository.BlobRepository +import com.tencent.bkrepo.repository.api.NodeClient +import org.springframework.stereotype.Service +import java.time.LocalDateTime + +@Service +class BlobService( + private val blobRepository: BlobRepository, + private val nodeClient: NodeClient, + private val storageManager: StorageManager +) { + fun create(blob: Blob): Blob { + with(blob) { + val userId = SecurityUtils.getUserId() + val now = LocalDateTime.now() + val tBlob = TDdcBlob( + id = null, + createdBy = userId, + createdDate = now, + lastModifiedBy = userId, + lastModifiedDate = now, + projectId = projectId, + repoName = repoName, + blobId = blobId.toString(), + contentId = contentId.toString(), + sha256 = sha256, + size = size + ) + blobRepository.replace(tBlob) + + return Blob.from(tBlob) + } + + } + + fun getBlob(projectId: String, repoName: String, blobId: String): Blob { + return blobRepository.findByBlobId(projectId, repoName, blobId)?.let { Blob.from(it) } + ?: throw BlobNotFoundException(projectId, repoName, blobId) + } + + fun findBlob(projectId: String, repoName: String, blobId: String): Blob? { + return blobRepository.findByBlobId(projectId, repoName, blobId)?.let { Blob.from(it) } + } + fun loadBlob(projectId: String, repoName: String, blobId: String): ArtifactInputStream { + val blob = blobRepository.findByBlobId(projectId, repoName, blobId) + ?: throw BlobNotFoundException(projectId, repoName, blobId) + return loadBlob(Blob.from(blob)) + } + + fun loadBlob(blob: Blob): ArtifactInputStream { + val repo = + ArtifactContextHolder.getRepoDetail(ArtifactContextHolder.RepositoryId(blob.projectId, blob.repoName)) + val node = nodeClient.getNodeDetail(blob.projectId, blob.repoName, blob.fullPath).data + ?: throw BlobNotFoundException(blob.projectId, blob.repoName, blob.blobId.toString()) + return storageManager.loadArtifactInputStream(node, repo.storageCredentials) + ?: throw BlobNotFoundException(blob.projectId, blob.repoName, blob.blobId.toString()) + } + + /** + * 根据blob大小排序，返回最小的blob + */ + fun getSmallestBlobByContentId(projectId: String, repoName: String, contentId: String): Blob? { + return blobRepository.findSmallestByContentId(projectId, repoName, contentId)?.let { Blob.from(it) } + } + + fun getBlobByBlobIds(projectId: String, repoName: String, blobIds: Collection): List { + return blobRepository.findByBlobIds(projectId, repoName, blobIds.toSet()).map { Blob.from(it) } + } + + fun addRefToBlobs(ref: Reference, blobIds: Set) { + blobRepository.addRefToBlob(ref.projectId, ref.repoName, ref.bucket, ref.key.toString(), blobIds) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/CompressedBlobService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/CompressedBlobService.kt new file mode 100644 index 0000000000..ce8aa4b115 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/CompressedBlobService.kt @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.service + +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext +import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.ddc.artifact.CompressedBlobArtifactInfo +import org.springframework.stereotype.Service + +@Service +class CompressedBlobService : ArtifactService() { + fun put(artifactInfo: CompressedBlobArtifactInfo, artifactFile: ArtifactFile) { + repository.upload(ArtifactUploadContext(artifactFile)) + } + + fun get(artifactInfo: CompressedBlobArtifactInfo) { + repository.download(ArtifactDownloadContext()) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt new file mode 100644 index 0000000000..3b9f2af0b2 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.service + +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext +import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo +import com.tencent.bkrepo.ddc.artifact.repository.DdcLocalRepository +import org.springframework.stereotype.Service + +@Service +class ReferenceArtifactService : ArtifactService() { + fun downloadRef(artifactInfo: ReferenceArtifactInfo) { + repository.download(ArtifactDownloadContext()) + } + + fun createRef(artifactInfo: ReferenceArtifactInfo, file: ArtifactFile) { + repository.upload(ArtifactUploadContext(file)) + } + + fun finalize(artifactInfo: ReferenceArtifactInfo) { + (ArtifactContextHolder.getRepository() as DdcLocalRepository).finalizeRef(artifactInfo) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceResolver.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceResolver.kt new file mode 100644 index 0000000000..ad7a28010d --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceResolver.kt @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.service + +import com.tencent.bkrepo.ddc.exception.BlobNotFoundException +import com.tencent.bkrepo.ddc.exception.NotImplementedException +import com.tencent.bkrepo.ddc.exception.ReferenceIsMissingBlobsException +import com.tencent.bkrepo.ddc.pojo.Blob +import com.tencent.bkrepo.ddc.pojo.ContentHash +import com.tencent.bkrepo.ddc.serialization.CbObject +import com.tencent.bkrepo.ddc.utils.isBinaryAttachment +import com.tencent.bkrepo.ddc.utils.isObjectAttachment +import org.springframework.stereotype.Service +import java.nio.ByteBuffer +import java.util.ArrayDeque + +@Service +class ReferenceResolver( + private val blobService: BlobService +) { + + /** + * 获取[cb]直接与间接引用的所有blob，保证blob存在 + * blob不存在时抛出[ReferenceIsMissingBlobsException] + */ + fun getReferencedBlobs(projectId: String, repoName: String, cb: CbObject): List { + val objectsToVisit = ArrayDeque() + val blobs = ArrayList() + val unresolvedBlobs = ArrayList() + objectsToVisit.offer(cb) + + while (objectsToVisit.isNotEmpty()) { + objectsToVisit.poll().iterateAttachments { field -> + val fieldAttachment = field.asAttachment() + if (field.isBinaryAttachment()) { + val contentId = fieldAttachment.toString() + blobService + .getSmallestBlobByContentId(projectId, repoName, contentId) + ?.let { blobs.add(it) } + ?: unresolvedBlobs.add(ContentHash(fieldAttachment.toByteArray())) + } else if (field.isObjectAttachment()) { + val contentHash = ContentHash(fieldAttachment.toByteArray()) + resolveObject(projectId, repoName, contentHash)?.let { + blobs.add(it.first) + objectsToVisit.offer(it.second) + } ?: unresolvedBlobs.add(contentHash) + } else { + throw NotImplementedException("Unknown attachment type for field $field") + } + } + } + + if (unresolvedBlobs.isNotEmpty()) { + throw ReferenceIsMissingBlobsException(unresolvedBlobs) + } + + return blobs + } + + /** + * 根据[blobId]获取其关联的数据，并转化为CbObject + * + * @return 找不到[blobId]关联的数据时返回null + */ + private fun resolveObject(projectId: String, repoName: String, blobId: ContentHash): Pair? { + return try { + val blob = blobService.getBlob(projectId, repoName, blobId.toString()) + val blobBytes = blobService.loadBlob(blob).readBytes() + val cb = CbObject(ByteBuffer.wrap(blobBytes)) + Pair(blob, cb) + } catch (_: BlobNotFoundException) { + null + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt new file mode 100644 index 0000000000..50836a0b82 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt @@ -0,0 +1,128 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.service + +import com.tencent.bkrepo.common.api.exception.BadRequestException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.ddc.config.DdcProperties +import com.tencent.bkrepo.ddc.exception.ReferenceIsMissingBlobsException +import com.tencent.bkrepo.ddc.model.TDdcRef +import com.tencent.bkrepo.ddc.pojo.ContentHash +import com.tencent.bkrepo.ddc.pojo.CreateRefResponse +import com.tencent.bkrepo.ddc.pojo.Reference +import com.tencent.bkrepo.ddc.repository.RefRepository +import com.tencent.bkrepo.ddc.serialization.CbObject +import com.tencent.bkrepo.ddc.utils.hasAttachments +import com.tencent.bkrepo.repository.api.NodeClient +import org.bson.types.Binary +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import java.nio.ByteBuffer +import java.time.LocalDateTime + +@Service +class ReferenceService( + private val ddcProperties: DdcProperties, + private val blobService: BlobService, + private val refResolver: ReferenceResolver, + private val refRepository: RefRepository, + private val nodeClient: NodeClient, + private val storageManager: StorageManager, +) { + fun create(ref: Reference): Reference { + val inlineBlob = if (ref.inlineBlob!!.size > ddcProperties.inlineBlobMaxSize.toBytes()) { + null + } else { + ref.inlineBlob + } + + val userId = SecurityUtils.getUserId() + val now = LocalDateTime.now() + val tRef = TDdcRef( + id = null, + createdBy = userId, + createdDate = now, + lastModifiedBy = userId, + lastModifiedDate = now, + lastAccessDate = now, // TODO 更新lastAccessDate + projectId = ref.projectId, + repoName = ref.repoName, + bucket = ref.bucket, + key = ref.key.toString(), + finalized = ref.finalized!!, + blobId = ref.blobId!!.toString(), + inlineBlob = Binary(inlineBlob), + expireDate = null // TODO 设置expireDate + ) + refRepository.replace(tRef) + return Reference.from(tRef) + } + + fun getReference( + projectId: String, + repoName: String, + bucket: String, + key: String, + includePayload: Boolean = true, + checkFinalized: Boolean = true, + ): Reference? { + val tRef = refRepository.find(projectId, repoName, bucket, key, includePayload) ?: return null + if (checkFinalized && !tRef.finalized) { + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, "Object ${tRef.bucket} ${tRef.key} is not finalized." + ) + } + + return Reference.from(tRef) + } + + fun finalize(ref: Reference, payload: ByteArray): CreateRefResponse { + val cbObject = CbObject(ByteBuffer.wrap(payload)) + var missingBlobs = emptyList() + if (cbObject.hasAttachments()) { + try { + val blobs = refResolver.getReferencedBlobs(ref.projectId, ref.repoName, cbObject) + blobService.addRefToBlobs(ref, blobs.mapTo(HashSet()) { it.toString() }) + } catch (e: ReferenceIsMissingBlobsException) { + missingBlobs = e.missingBlobs + } + } + + if (missingBlobs.isEmpty()) { + refRepository.finalize(ref.projectId, ref.repoName, ref.bucket, ref.key.toString()) + } + + return CreateRefResponse((missingBlobs).mapTo(HashSet()) { it.toString() }) + } + + companion object { + private val logger = LoggerFactory.getLogger(ReferenceService::class.java) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/BlakeUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/BlakeUtils.kt new file mode 100644 index 0000000000..93c0d0c13b --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/BlakeUtils.kt @@ -0,0 +1,128 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.utils + +import com.tencent.bkrepo.common.artifact.resolve.file.stream.StreamArtifactFile +import com.tencent.bkrepo.ddc.serialization.IoHash.Companion.NUM_BYTES +import org.bouncycastle.crypto.digests.Blake3Digest +import org.bouncycastle.util.encoders.Hex +import java.io.FilterInputStream +import java.io.IOException +import java.io.InputStream +import java.nio.ByteBuffer + +object BlakeUtils { + const val OUT_LEN = NUM_BYTES + val ZERO = ByteBuffer.allocate(OUT_LEN) + fun hash(content: String): ByteArray { + return hash(content.toByteArray()) + } + + fun hash(bytes: ByteArray): ByteArray { + val digest = Blake3Digest(OUT_LEN) + digest.update(bytes, 0, bytes.size) + val hashCode = ByteArray(digest.digestSize) + digest.doFinal(hashCode, 0) + return hashCode + } + + fun hash(byteBuffers: List): ByteArray { + val digest = Blake3Digest(OUT_LEN) + byteBuffers.forEach { buffer -> + digest.update(buffer.array(), buffer.arrayOffset(), buffer.remaining()) + } + val hashCode = ByteArray(digest.digestSize) + digest.doFinal(hashCode, 0) + return hashCode + } + + fun ByteArray.hex(): String = Hex.toHexString(this) + + fun ByteBuffer.hex(): String { + val arr = ByteBuffer.allocate(remaining()).put(duplicate()).array() + return Hex.toHexString(arr, 0, arr.size) + } + + fun InputStream.toBlake3InputStream() = Blake3InputStream(this) + + // TODO 改为在读文件时就计算哈希，避免重复读流 + fun StreamArtifactFile.blake3(): ByteArray { + val digest = Blake3Digest(OUT_LEN) + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + getInputStream().use { + var size = it.read(buffer) + while (size != -1) { + digest.update(buffer, 0, size) + size = it.read(buffer) + } + } + val hashCode = ByteArray(digest.digestSize) + digest.doFinal(hashCode, 0) + return hashCode + } + + class Blake3InputStream(inputStream: InputStream) : FilterInputStream(inputStream) { + + private val digest by lazy { Blake3Digest(OUT_LEN) } + + @Throws(IOException::class) + override fun read(): Int { + val b = `in`.read() + if (b != -1) { + digest.update(b.toByte()) + } + return b + } + + @Throws(IOException::class) + override fun read(bytes: ByteArray, off: Int, len: Int): Int { + val numOfBytesRead = `in`.read(bytes, off, len) + if (numOfBytesRead != -1) { + digest.update(bytes, off, numOfBytesRead) + } + return numOfBytesRead + } + + override fun markSupported(): Boolean { + return false + } + + override fun mark(readlimit: Int) {} + + @Throws(IOException::class) + override fun reset() { + throw IOException("reset not supported") + } + + fun hash(): ByteArray { + val hashCode = ByteArray(digest.digestSize) + digest.doFinal(hashCode, 0) + return hashCode + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/ByteBufferUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/ByteBufferUtils.kt new file mode 100644 index 0000000000..54cb9184bb --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/ByteBufferUtils.kt @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.utils + +import java.nio.ByteBuffer + +object ByteBufferUtils { + val EMPTY = ByteBuffer.allocate(0) + + fun byteBufferOf(vararg elements: Byte) = ByteBuffer.wrap(elements) +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbFieldUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbFieldUtils.kt new file mode 100644 index 0000000000..c7dcf9ebf7 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbFieldUtils.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.utils + +import com.tencent.bkrepo.ddc.serialization.CbField +import com.tencent.bkrepo.ddc.serialization.CbFieldUtils +import com.tencent.bkrepo.ddc.serialization.CbObject + +fun CbField.hasName() = CbFieldUtils.hasFieldName(typeWithFlags) + +fun CbField.isNull() = CbFieldUtils.isNull(typeWithFlags) + +fun CbField.isObject() = CbFieldUtils.isObject(typeWithFlags) + +fun CbField.isArray() = CbFieldUtils.isArray(typeWithFlags) + +fun CbField.isString() = CbFieldUtils.isString(typeWithFlags) + +fun CbField.isBinary() = CbFieldUtils.isBinary(typeWithFlags) + +fun CbField.isInteger() = CbFieldUtils.isInteger(typeWithFlags) + +fun CbField.isFloat() = CbFieldUtils.isFloat(typeWithFlags) + +fun CbField.isBool() = CbFieldUtils.isBool(typeWithFlags) + +fun CbField.isObjectAttachment() = CbFieldUtils.isObjectAttachment(typeWithFlags) +fun CbField.isBinaryAttachment() = CbFieldUtils.isBinaryAttachment(typeWithFlags) +fun CbField.isAttachment() = CbFieldUtils.isAttachment(typeWithFlags) +fun CbField.isHash() = CbFieldUtils.isHash(typeWithFlags) +fun CbField.isUuid() = CbFieldUtils.isUuid(typeWithFlags) + +fun CbField.hasAttachments(): Boolean { + return if (isAttachment()) { + true + } else { + any { it.isAttachment() || it.isObject() && it.hasAttachments() || it.isArray() && it.hasAttachments() } + } +} + +fun CbObject.hasAttachments(): Boolean { + return any { it.hasAttachments() } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbWriterUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbWriterUtils.kt new file mode 100644 index 0000000000..1be70035ff --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/CbWriterUtils.kt @@ -0,0 +1,126 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.utils + +import com.tencent.bkrepo.ddc.serialization.CbFieldType +import com.tencent.bkrepo.ddc.serialization.CbWriterBase +import com.tencent.bkrepo.ddc.serialization.VarULong +import com.tencent.bkrepo.ddc.utils.BlakeUtils.OUT_LEN +import java.nio.ByteBuffer + +fun CbWriterBase.measureFieldWithLength(length: Int) = length + VarULong.measureUnsigned(length.toLong()); + +fun CbWriterBase.writeFieldWithLength(type: CbFieldType, name: String?, length: Int): ByteBuffer { + val fullLength = measureFieldWithLength(length) + val buffer = writeField(type, name, fullLength) + VarULong.writeUnsigned(buffer, length.toLong()) + return buffer +} + +fun CbWriterBase.beginUniformArray(elementType: CbFieldType) = beginUniformArray(null, elementType); + +fun CbWriterBase.beginUniformArray(name: String? = null, elementType: CbFieldType) = beginArray(name, elementType); + +fun CbWriterBase.writeNull(name: String? = null) = writeField(CbFieldType.Null, name, 0) + +fun CbWriterBase.writeNullValue() = writeField(CbFieldType.Null, null, 0) + +fun CbWriterBase.writeBoolValue(value: Boolean) = writeBool(null, value) + +fun CbWriterBase.writeBool(name: String? = null, value: Boolean) = + writeField(if (value) CbFieldType.BoolTrue else CbFieldType.BoolFalse, name, 0) + +fun CbWriterBase.writeIntegerValue(value: Int) = writeLong(null, value.toLong()) +fun CbWriterBase.writeInteger(name: String, value: Int) = writeLong(name, value.toLong()) + +fun CbWriterBase.writeLongValue(value: Long) = writeLong(null, value) + +fun CbWriterBase.writeLong(name: String? = null, value: Long) { + if (value >= 0) { + val length = VarULong.measureUnsigned(value) + val data = writeField(CbFieldType.IntegerPositive, name, length) + VarULong.writeUnsigned(data, value) + } else { + // TODO 直接取反可能存在溢出的问题 + val length = VarULong.measureUnsigned(-value) + val data = writeField(CbFieldType.IntegerNegative, name, length) + VarULong.writeUnsigned(data, -value) + } +} + +fun CbWriterBase.writeDoubleValue(value: Double) = writeDouble(null, value) + +fun CbWriterBase.writeDouble(name: String? = null, value: Double) { + val buffer = writeField(CbFieldType.Float64, name, Double.SIZE_BYTES) + buffer.putDouble(value) +} + +fun CbWriterBase.writeHashValue(value: ByteArray) = writeHash(null, value) + +fun CbWriterBase.writeHash(name: String? = null, value: ByteArray) { + val buffer = writeField(CbFieldType.Hash, name, OUT_LEN) + buffer.put(value) +} + +fun CbWriterBase.writeBinaryAttachmentValue(hash: ByteArray) = writeBinaryAttachment(null, hash) + +fun CbWriterBase.writeBinaryAttachment(name: String? = null, hash: ByteArray) { + val buffer = writeField(CbFieldType.BinaryAttachment, name, OUT_LEN) + buffer.put(hash) +} + +fun CbWriterBase.writeObjectAttachmentValue(hash: ByteArray) = writeObjectAttachment(null, hash) + +fun CbWriterBase.writeObjectAttachment(name: String? = null, hash: ByteArray) { + writeField(CbFieldType.ObjectAttachment, name, OUT_LEN).put(hash) +} + +fun CbWriterBase.writeStringValue(value: String) = writeString(null, value) + +fun CbWriterBase.writeString(name: String? = null, value: String?) { + if (value != null) { + writeFieldWithLength(CbFieldType.String, name, value.length).put(value.toByteArray()) + } +} + +fun CbWriterBase.writeBinaryReference(name: String? = null, data: ByteBuffer) { + val length = VarULong.measureUnsigned(data.remaining().toLong()) + val buffer = writeField(CbFieldType.Binary, name, length) + VarULong.writeUnsigned(buffer, data.remaining().toLong()) + writeReference(data.asReadOnlyBuffer()) +} + +fun CbWriterBase.writeBinaryValue(value: ByteBuffer) = writeBinary(null, value) + +fun CbWriterBase.writeBinary(name: String? = null, value: ByteBuffer) { + writeFieldWithLength(CbFieldType.Binary, name, value.remaining()).put(value.asReadOnlyBuffer()) +} + +fun CbWriterBase.writeBinaryArrayValue(value: ByteArray) = writeBinaryValue(ByteBuffer.wrap(value)) + +fun CbWriterBase.writeBinaryArray(name: String, value: ByteArray) = writeBinary(name, ByteBuffer.wrap(value)) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/DdcUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/DdcUtils.kt new file mode 100644 index 0000000000..41f4ffbde4 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/DdcUtils.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.utils + +import com.tencent.bkrepo.ddc.model.TDdcBlob +import com.tencent.bkrepo.ddc.model.TDdcRef + +object DdcUtils { + const val DIR_BLOBS = "blobs" + + fun TDdcRef.fullPath() = "/$bucket/$key" + + fun TDdcBlob.fullPath() = "/blobs/$blobId" +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/MediaTypeUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/MediaTypeUtils.kt new file mode 100644 index 0000000000..97f74ef7f5 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/MediaTypeUtils.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.utils + +/** + * compact binary + */ +const val MEDIA_TYPE_UNREAL_COMPACT_BINARY = "application/x-ue-cb"; + +/** + * compressed buffers + */ +const val MEDIA_TYPE_UNREAL_UNREAL_COMPRESSED_BUFFER = "application/x-ue-comp"; + +const val MEDIA_TYPE_JUPITER_INLINED_PAYLOAD = "application/x-jupiter-inline" + +/** + * compact binary packages + */ +const val MEDIA_TYPE_UNREAL_COMPACT_BINARY_PACKAGE = "application/x-ue-cbpkg" diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/NodeUtils.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/NodeUtils.kt new file mode 100644 index 0000000000..406c9597cb --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/utils/NodeUtils.kt @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.utils + +const val NODE_METADATA_KEY_PREFIX = "ddc_" + +const val NODE_METADATA_KEY_BLOB_ID = "${NODE_METADATA_KEY_PREFIX}blob_id" +const val NODE_METADATA_KEY_CONTENT_ID = "${NODE_METADATA_KEY_PREFIX}content_id" diff --git a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldTest.kt b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldTest.kt new file mode 100644 index 0000000000..15539c5d64 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbFieldTest.kt @@ -0,0 +1,1197 @@ +package com.tencent.bkrepo.ddc.serialization + +import com.tencent.bkrepo.ddc.serialization.CbField.CbFieldError.None +import com.tencent.bkrepo.ddc.serialization.CbField.CbFieldError.RangeError +import com.tencent.bkrepo.ddc.serialization.CbField.CbFieldError.TypeError +import com.tencent.bkrepo.ddc.utils.BlakeUtils +import com.tencent.bkrepo.ddc.utils.ByteBufferUtils +import com.tencent.bkrepo.ddc.utils.ByteBufferUtils.byteBufferOf +import com.tencent.bkrepo.ddc.utils.hasName +import com.tencent.bkrepo.ddc.utils.isArray +import com.tencent.bkrepo.ddc.utils.isBinary +import com.tencent.bkrepo.ddc.utils.isBinaryAttachment +import com.tencent.bkrepo.ddc.utils.isBool +import com.tencent.bkrepo.ddc.utils.isFloat +import com.tencent.bkrepo.ddc.utils.isHash +import com.tencent.bkrepo.ddc.utils.isInteger +import com.tencent.bkrepo.ddc.utils.isNull +import com.tencent.bkrepo.ddc.utils.isObject +import com.tencent.bkrepo.ddc.utils.isObjectAttachment +import com.tencent.bkrepo.ddc.utils.isString +import com.tencent.bkrepo.ddc.utils.isUuid +import com.tencent.bkrepo.ddc.utils.writeBinaryAttachment +import com.tencent.bkrepo.ddc.utils.writeBinaryAttachmentValue +import com.tencent.bkrepo.ddc.utils.writeObjectAttachment +import com.tencent.bkrepo.ddc.utils.writeString +import com.tencent.bkrepo.ddc.utils.writeStringValue +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.nio.ByteBuffer +import java.util.UUID +import kotlin.experimental.and +import kotlin.experimental.or + +class CbFieldTest { + @Test + fun noneTest() { + // Test CbField() + val defaultField = CbField() + assertFalse(CbFieldUtils.hasFieldName(defaultField.typeWithFlags)) + assertFalse(defaultField.hasValue()) + assertFalse(defaultField.hasError()) + assertTrue(defaultField.error == None) + assertEquals(defaultField.getSize(), 1) + assertEquals(defaultField.name.length, 0) + assertFalse(defaultField.hasName()) + assertFalse(defaultField.hasValue()) + assertFalse(defaultField.hasError()) + assertEquals(defaultField.error, None) + assertEquals( + defaultField.getHash(), + Blake3Hash.compute(ByteBuffer.wrap(ByteArray(1) { CbFieldType.None.value })) + ) + assertFalse(defaultField.tryGetView().first) + + // Test CbField(None) + var noneField = CbField(ByteBufferUtils.EMPTY, CbFieldType.None) + assertEquals(noneField.getSize(), 1) + assertEquals(noneField.name.length, 0) + assertFalse(noneField.hasName()) + assertFalse(noneField.hasValue()) + assertFalse(noneField.hasError()) + assertEquals(noneField.error, None) + assertEquals(noneField.getHash(), CbField().getHash()) + assertFalse(noneField.tryGetView().first) + + // Test CbField(None|Type|Name) + var fieldType = CbFieldType.None.value or CbFieldType.HasFieldName.value + var noneBytes = byteBufferOf(fieldType, 4, 'N'.toByte(), 'a'.toByte(), 'm'.toByte(), 'e'.toByte()) + + noneField = CbField(noneBytes) + assertEquals(noneField.getSize(), noneBytes.remaining()) + assertEquals(noneField.name, "Name") + assertTrue(noneField.hasName()) + assertFalse(noneField.hasValue()) + assertEquals(noneField.getHash(), Blake3Hash.compute(noneBytes)) + var view = noneField.tryGetView() + assertTrue(view.first && view.second == noneBytes) + + var copyBytes = ByteBuffer.allocate(noneBytes.remaining()) + noneField.copyTo(copyBytes) + copyBytes.flip() + assertTrue(noneBytes == copyBytes) + + + // Test CbField(None|Type) + fieldType = CbFieldType.None.value + noneBytes = byteBufferOf(fieldType) + noneField = CbField(noneBytes) + assertEquals(noneField.getSize(), noneBytes.remaining()) + assertEquals(noneField.name.length, 0) + assertFalse(noneField.hasName()) + assertFalse(noneField.hasValue()) + assertEquals(noneField.getHash(), CbField().getHash()) + view = noneField.tryGetView() + assertTrue(view.first && view.second == noneBytes) + + + // Test CbField(None|Name) + fieldType = CbFieldType.None.value or CbFieldType.HasFieldName.value + noneBytes = byteBufferOf(fieldType, 4, 'N'.toByte(), 'a'.toByte(), 'm'.toByte(), 'e'.toByte()) + + var b = noneBytes.asReadOnlyBuffer() + b.position(1) + noneField = CbField(b.slice(), fieldType) + assertEquals(noneField.getSize(), noneBytes.remaining()) + assertEquals(noneField.name, "Name") + assertTrue(noneField.hasName()) + assertFalse(noneField.hasValue()) + assertEquals(noneField.getHash(), Blake3Hash.compute(noneBytes)) + + view = noneField.tryGetView() + assertFalse(view.first) + + copyBytes = ByteBuffer.allocate(noneBytes.remaining()) + noneField.copyTo(copyBytes) + copyBytes.flip() + assertTrue(noneBytes == copyBytes) + + // Test CbField(None|EmptyName) + fieldType = CbFieldType.None.value or CbFieldType.HasFieldName.value + noneBytes = byteBufferOf(fieldType, 0) + + b = noneBytes.asReadOnlyBuffer() + b.position(1) + noneField = CbField(b.slice(), fieldType) + assertEquals(noneField.getSize(), noneBytes.remaining()) + assertEquals(noneField.name, "") + assertTrue(noneField.hasName()) + assertFalse(noneField.hasValue()) + assertEquals(noneField.getHash(), Blake3Hash.compute(noneBytes)) + + view = noneField.tryGetView() + assertFalse(view.first) + } + + @Test + fun nullTest() { + // Test CbField(Null) + val nullField = CbField(ByteBufferUtils.EMPTY, CbFieldType.Null) + assertEquals(nullField.getSize(), 1) + assertTrue(nullField.isNull()) + assertTrue(nullField.hasValue()) + assertFalse(nullField.hasError()) + assertEquals(nullField.error, None) + assertEquals(nullField.getHash(), Blake3Hash.compute(byteBufferOf(CbFieldType.Null.value))) + + // Test CbField(None) as Null + val field = CbField() + assertFalse(field.isNull()) + } + + @Test + fun objectTest() { + // Test CbField(Object, Empty) + testField(CbFieldType.Object, ByteBuffer.wrap(ByteArray(1))) + + // Test CbField(Object, Empty) + var obj = CbObject.EMPTY + testIntObject(obj, 0, 1) + + // Find fields that do not exist. + assertFalse(obj.find("Field").hasValue()) + assertFalse(obj.findIgnoreCase("Field").hasValue()) + assertFalse(obj["Field"].hasValue()) + + // Advance an iterator past the last field. + val iterator = obj.createIterator() + assertFalse(iterator.hasNext()) + assertTrue(!iterator.hasNext()) + + + // Test CbField(Object, NotEmpty) + val intType: Byte = (CbFieldType.HasFieldName.value or CbFieldType.IntegerPositive.value) + var payload = byteBufferOf( + 12, + intType, 1, 'A'.toByte(), 1, + intType, 1, 'B'.toByte(), 2, + intType, 1, 'C'.toByte(), 3 + ) + var field = CbField(payload.asReadOnlyBuffer(), CbFieldType.Object) + testField(CbFieldType.Object, field, CbObject(payload, CbFieldType.Object)) + obj = field.asObject() + testIntObject(obj, 3, payload.remaining()) + testIntObject(field.asObject(), 3, payload.remaining()) + assertTrue(obj.equals(field.asObject())) + assertEquals(obj.find("B").asInt32(), 2) + assertEquals(obj.find("b").asInt32(4), 4) + assertEquals(obj.findIgnoreCase("B").asInt32(), 2) + assertEquals(obj.findIgnoreCase("b").asInt32(), 2) + assertEquals(obj["B"].asInt32(), 2) + assertEquals(obj["b"].asInt32(4), 4) + + + // Test CbField(UniformObject, NotEmpty) + payload = byteBufferOf( + 10, intType, + 1, 'A'.toByte(), 1, + 1, 'B'.toByte(), 2, + 1, 'C'.toByte(), 3 + ) + field = CbField(payload.asReadOnlyBuffer(), CbFieldType.UniformObject) + testField(CbFieldType.UniformObject, field, CbObject(payload, CbFieldType.UniformObject)) + obj = field.asObject() + testIntObject(obj, 3, payload.remaining()) + testIntObject(field.asObject(), 3, payload.remaining()) + assertTrue(obj == field.asObject()) + assertEquals(obj.find("B").asInt32(), 2) + assertEquals(obj.find("b").asInt32(4), 4) + assertEquals(obj.findIgnoreCase("B").asInt32(), 2) + assertEquals(obj.findIgnoreCase("b").asInt32(), 2) + assertEquals(obj["B"].asInt32(), 2) + assertEquals(obj["b"].asInt32(4), 4) + + // Equals + val namedPayload = byteBufferOf( + 1, 'O'.toByte(), + 10, intType, + 1, 'A'.toByte(), 1, + 1, 'B'.toByte(), 2, + 1, 'C'.toByte(), 3 + ) + val namedField = CbField(namedPayload, CbFieldType.UniformObject.value or CbFieldType.HasFieldName.value) + assertTrue(field.asObject() == namedField.asObject()) + + // CopyTo + val copyBytes = ByteBuffer.allocate(payload.remaining() + 1) + field.asObject().copyTo(copyBytes) + copyBytes.position(1) + assertTrue(payload == copyBytes) + + copyBytes.clear() + namedField.asObject().copyTo(copyBytes) + copyBytes.position(1) + assertTrue(payload == copyBytes) + + // Test CbField(None) as Object + field = CbField.EMPTY + testField(fieldType = CbFieldType.Object, field = field, expectedError = TypeError) + + // Test FCbObjectView(ObjectWithName) and CreateIterator + val objectType = (CbFieldType.Object.value or CbFieldType.HasFieldName.value) + val buffer = byteBufferOf( + objectType, + 3, 'K'.toByte(), 'e'.toByte(), 'y'.toByte(), + 4, + (CbFieldType.HasFieldName.value or CbFieldType.IntegerPositive.value), + 1, 'F'.toByte(), + 8 + ) + var o = CbObject(buffer) + assertEquals(o.getSize(), 6) + for (f in o) { + assertEquals(f.name, "F") + assertEquals(f.asInt32(), 8) + } + + // Test FCbObjectView as CbFieldIterator + var count = 0 + o = CbObject.EMPTY + for (f in CbFieldIterator.makeSingle(o.asField())) { + assertTrue(f.isObject()) + ++count + } + assertEquals(count, 1) + } + + @Test + fun arrayTest() { + // Test CbField(Array, Empty) + testField(CbFieldType.Array, byteBufferOf(1, 0)) + + // Test CbField(Array, Empty) + var array = CbArray() + testIntArray(array, 0, 2) + + // Advance an iterator past the last field. + val iterator = array.createIterator() + assertFalse(iterator.hasNext()) + for (count in 16 downTo 1) { + iterator.next().asInt32() + } + assertFalse(iterator.hasNext()) + + // Test CbField(Array, NotEmpty) + val intType: Byte = CbFieldType.IntegerPositive.value + var payload = byteBufferOf(7, 3, intType, 1, intType, 2, intType, 3) + var field = CbField(payload, CbFieldType.Array) + testField(CbFieldType.Array, field, CbArray(payload, CbFieldType.Array)) + array = field.asArray() + testIntArray(array, 3, payload.remaining()) + testIntArray(field.asArray(), 3, payload.remaining()) + assertTrue(array == field.asArray()) + + // Test CbField(UniformArray) + payload = byteBufferOf(5, 3, intType, 1, 2, 3) + field = CbField(payload, CbFieldType.UniformArray) + testField(CbFieldType.UniformArray, field, CbArray(payload, CbFieldType.UniformArray)) + array = field.asArray() + testIntArray(array, 3, payload.remaining()) + testIntArray(field.asArray(), 3, payload.remaining()) + assertTrue(array == field.asArray()) + + // Equals + val namedPayload = byteBufferOf(1, 'A'.toByte(), 5, 3, intType, 1, 2, 3) + val namedField = CbField(namedPayload, CbFieldType.UniformArray.value or CbFieldType.HasFieldName.value) + assertTrue(field.asArray() == namedField.asArray()) + assertTrue(field == field.asArray().asField()) + assertTrue(namedField == namedField.asArray().asField()) + + // CopyTo + val copyBytes = ByteBuffer.allocate(payload.remaining() + CbFieldType.SIZE_OF_CB_FIELD_TYPE) + field.asArray().copyTo(copyBytes) + copyBytes.position(1) + assertTrue(payload == copyBytes) + + copyBytes.clear() + namedField.asArray().copyTo(copyBytes) + copyBytes.position(1) + assertTrue(payload == copyBytes) + + // TryGetView + assertFalse(field.asArray().tryGetView().first) + assertFalse(namedField.asArray().tryGetView().first) + + // Test CbField(None) as Array + field = CbField() + testField(fieldType = CbFieldType.Array, field = field, expectedError = TypeError) + + + // Test CbArray(ArrayWithName) and CreateIterator + val arrayType: Byte = (CbFieldType.Array.value or CbFieldType.HasFieldName.value) + val buffer = byteBufferOf( + arrayType, + 3, 'K'.toByte(), 'e'.toByte(), 'y'.toByte(), + 3, 1, CbFieldType.IntegerPositive.value, 8 + ) + array = CbArray(buffer) + assertEquals(array.getSize(), 5) + for (f in array) { + assertEquals(f.asInt32(), 8) + } + + + // Test CbArray as CbFieldIterator + var count = 0 + array = CbArray() + for (f in CbFieldIterator.makeSingle(array.asField())) { + assertTrue(f.isArray()) + ++count + } + assertEquals(count, 1) + } + + @Test + fun binaryTest() { + // Test CbField(Binary, Empty) + testField(CbFieldType.Binary, byteBufferOf(0)) + + // Test CbField(Binary, Value) + run { + val payload = byteBufferOf(3, 4, 5, 6) // Size: 3, Data: 4/5/6 + val fieldView = CbField(payload.asReadOnlyBuffer(), CbFieldType.Binary) + payload.position(1) + testField(CbFieldType.Binary, fieldView, payload.slice()) + } + + // Test CbField(None) as Binary + run { + val fieldView = CbField() + val default = byteBufferOf(1, 2, 3) + testField( + fieldType = CbFieldType.Binary, + field = fieldView, + expectedError = TypeError, + expectedValue = default, + defaultValue = default, + ) + } + } + + @Test + fun stringTest() { + // Test CbField(String, Empty) + testField(CbFieldType.String, byteBufferOf(0)) + + // Test CbField(String, Value) + run { + val payload = byteBufferOf(3, 'A'.toByte(), 'B'.toByte(), 'C'.toByte()) // Size: 3, Data: ABC + testField(CbFieldType.String, payload, "ABC") + } + + // Test CbField (String, OutOfRangeSize) + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, (1L shl 31)) + payload.flip() + testField( + fieldType = CbFieldType.String, + payload = payload, + expectedError = RangeError, + defaultValue = "ABC", + expectedValue = "ABC", + ) + } + + // Test CbField(None) as String + run { + val field = CbField() + testField( + fieldType = CbFieldType.String, + field = field, + expectedError = TypeError, + defaultValue = "ABC", + expectedValue = "ABC", + ) + } + } + + @Test + fun integerTest() { + // Test CbField(IntegerPositive) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos7, 0x00) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos7, 0x7f) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos8, 0x80) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos8, 0xff) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos15, 0x0100) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos15, 0x7fff) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos16, 0x8000) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos16, 0xffff) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos31, 0x0001_0000) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos31, 0x7fff_ffff) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos32, 0x8000_0000) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos32, 0xffff_ffff) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos63, 0x0000_0001_0000_0000) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos63, 0x7fff_ffff_ffff_ffff) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos64, Long.MIN_VALUE) + testIntegerField(CbFieldType.IntegerPositive, EIntType.Pos64, -1L) + + // Test CbField(IntegerNegative) + testIntegerField(CbFieldType.IntegerNegative, EIntType.Neg7, 0x01) + testIntegerField(CbFieldType.IntegerNegative, EIntType.Neg7, 0x80) + testIntegerField(CbFieldType.IntegerNegative, EIntType.Neg15, 0x81) + testIntegerField(CbFieldType.IntegerNegative, EIntType.Neg15, 0x8000) + testIntegerField(CbFieldType.IntegerNegative, EIntType.Neg31, 0x8001) + testIntegerField(CbFieldType.IntegerNegative, EIntType.Neg31, 0x8000_0000) + testIntegerField(CbFieldType.IntegerNegative, EIntType.Neg63, 0x8000_0001) + testIntegerField(CbFieldType.IntegerNegative, EIntType.Neg63, Long.MIN_VALUE) + testIntegerField(CbFieldType.IntegerNegative, EIntType.None, Long.MIN_VALUE + 1) + testIntegerField(CbFieldType.IntegerNegative, EIntType.None, -1L) + + // Test CbField(None) as Integer + val field = CbField() + testField( + fieldType = CbFieldType.IntegerPositive, + field = field, + expectedError = TypeError, + expectedValue = 8L, + defaultValue = 8L + ) + testField( + fieldType = CbFieldType.IntegerNegative, + field = field, + expectedError = TypeError, + expectedValue = 8L, + defaultValue = 8L + ) + } + + @Test + fun floatTest() { + // Test CbField(Float, 32-bit) + run { + val payload = byteBufferOf(0xc0.toByte(), 0x12, 0x34, 0x56) // -2.28444433f + // TODO 存在精度问题 + testField(CbFieldType.Float32, payload, -2.2844443f) + val field = CbField(payload, CbFieldType.Float32) + testField(CbFieldType.Float64, field, -2.2844443321228027) + } + + // Test CbField(Float, 64-bit) + run { + val payload = byteBufferOf( + 0xc1.toByte(), + 0x23, + 0x45, + 0x67, + 0x89.toByte(), + 0xab.toByte(), + 0xcd.toByte(), + 0xef.toByte() + ) // -631475.76888888876 + testField(CbFieldType.Float64, payload, -631475.76888888876) + val field = CbField(payload, CbFieldType.Float64) + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = RangeError, + expectedValue = 8.0f, + defaultValue = 8.0f, + ) + } + + // Test CbField(Integer+, MaxBinary32) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, (1L shl 24) - 1) // 16,777,215 + val field = CbField(payload, CbFieldType.IntegerPositive) + testField(CbFieldType.Float32, field, 16_777_215.0f) + testField(CbFieldType.Float64, field, 16_777_215.0) + } + + // Test CbField(Integer+, MaxBinary32+1) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, 1L shl 24) // 16,777,216 + val field = CbField(payload, CbFieldType.IntegerPositive) + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = RangeError, + expectedValue = 8.0f, + defaultValue = 8.0f + ) + testField(CbFieldType.Float64, field, 16_777_216.0) + } + + // Test CbField(Integer+, MaxBinary64) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, (1L shl 53) - 1) // 9,007,199,254,740,991 + val field = CbField(payload, CbFieldType.IntegerPositive) + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = RangeError, + expectedValue = 8.0f, + defaultValue = 8.0f + ) + testField(CbFieldType.Float64, field, 9_007_199_254_740_991.0) + } + + // Test CbField(Integer+, MaxBinary64+1) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, 1L shl 53) // 9,007,199,254,740,992 + val field = CbField(payload, CbFieldType.IntegerPositive) + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = RangeError, + expectedValue = 8.0f, + defaultValue = 8.0f + ) + testField( + fieldType = CbFieldType.Float64, + field = field, + expectedError = RangeError, + expectedValue = 8.0, + defaultValue = 8.0 + ) + } + + // Test CbField(Integer+, MaxUInt64) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, (0L.inv())) // Max uint64 + val field = CbField(payload, CbFieldType.IntegerPositive) + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = RangeError, + expectedValue = 8.0f, + defaultValue = 8.0f + ) + testField( + fieldType = CbFieldType.Float64, + field = field, + expectedError = RangeError, + expectedValue = 8.0, + defaultValue = 8.0 + ) + } + + // Test CbField(Integer-, MaxBinary32) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, (1L shl 24) - 2) // -16,777,215 + val field = CbField(payload, CbFieldType.IntegerNegative) + testField(CbFieldType.Float32, field, -16_777_215.0f) + testField(CbFieldType.Float64, field, -16_777_215.0) + } + + // Test CbField(Integer-, MaxBinary32+1) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, (1L shl 24) - 1) // -16,777,216 + val field = CbField(payload, CbFieldType.IntegerNegative) + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = RangeError, + expectedValue = 8.0f, + defaultValue = 8.0f + ) + testField(CbFieldType.Float64, field, -16_777_216.0) + } + +// Test CbField(Integer-, MaxBinary64) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, (1L shl 53) - 2) // -9,007,199,254,740,991 + val field = CbField(payload, CbFieldType.IntegerNegative) + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = RangeError, + expectedValue = 8.0f, + defaultValue = 8.0f + ) + testField(CbFieldType.Float64, field, -9_007_199_254_740_991.0) + } + +// Test CbField(Integer-, MaxBinary64+1) as Float + run { + val payload = ByteBuffer.allocate(9) + VarULong.writeUnsigned(payload, (1L shl 53) - 1) // -9,007,199,254,740,992 + val field = CbField(payload, CbFieldType.IntegerNegative) + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = RangeError, + expectedValue = 8.0f, + defaultValue = 8.0f + ) + testField( + fieldType = CbFieldType.Float64, + field = field, + expectedError = RangeError, + expectedValue = 8.0, + defaultValue = 8.0 + ) + } + +// Test CbField(None) as Float + run { + val field = CbField() + testField( + fieldType = CbFieldType.Float32, + field = field, + expectedError = TypeError, + expectedValue = 8.0f, + defaultValue = 8.0f + ) + testField( + fieldType = CbFieldType.Float64, + field = field, + expectedError = TypeError, + expectedValue = 8.0, + defaultValue = 8.0 + ) + } + } + + @Test + fun boolTest() { + // Test CbField(Bool, False) + testField(CbFieldType.BoolFalse, ByteBufferUtils.EMPTY, expectedValue = false, defaultValue = true) + + // Test CbField(Bool, True) + testField(CbFieldType.BoolTrue, ByteBufferUtils.EMPTY, expectedValue = true, defaultValue = false) + + // Test CbField(None) as Bool + val defaultField = CbField() + testField( + fieldType = CbFieldType.BoolFalse, + field = defaultField, + expectedError = TypeError, + expectedValue = false, + defaultValue = false + ) + testField( + fieldType = CbFieldType.BoolTrue, + field = defaultField, + expectedError = TypeError, + expectedValue = true, + defaultValue = true + ) + } + + @Test + fun objectAttachmentTest() { + val zeroBytes = ByteBuffer.allocate(20) + val sequentialBytes = byteBufferOf( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 + ) + + // Test CbField(ObjectAttachment, Zero) + testField(fieldType = CbFieldType.ObjectAttachment, payload = zeroBytes) + + // Test CbField(ObjectAttachment, NonZero) + testField( + CbFieldType.ObjectAttachment, + sequentialBytes, + CbObjectAttachment(IoHash(sequentialBytes.asReadOnlyBuffer())) + ) + + // Test CbField(ObjectAttachment, NonZero) AsAttachment + sequentialBytes.clear() + val field = CbField(sequentialBytes, CbFieldType.ObjectAttachment) + testField( + CbFieldType.ObjectAttachment, + field, + CbObjectAttachment(IoHash(sequentialBytes.asReadOnlyBuffer())), + CbObjectAttachment(IoHash.ZERO), + None + ) + + // Test CbField(None) as ObjectAttachment + val defaultField = CbField() + testField( + fieldType = CbFieldType.ObjectAttachment, + field = defaultField, + expectedError = TypeError, + defaultValue = CbObjectAttachment(IoHash(sequentialBytes.asReadOnlyBuffer())), + expectedValue = CbObjectAttachment(IoHash(sequentialBytes.asReadOnlyBuffer())) + ) + } + + @Test + fun binaryAttachmentTest() { + val zeroBytes = ByteBuffer.allocate(20) + val sequentialBytes = byteBufferOf( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 + ) + + // Test CbField(BinaryAttachment, Zero) + testField(CbFieldType.BinaryAttachment, zeroBytes) + + // Test CbField(BinaryAttachment, NonZero) + testField( + CbFieldType.BinaryAttachment, + sequentialBytes, + CbBinaryAttachment(IoHash(sequentialBytes.asReadOnlyBuffer())) + ) + + // Test CbField(BinaryAttachment, NonZero) AsAttachment + val field = CbField(sequentialBytes, CbFieldType.BinaryAttachment) + testField( + CbFieldType.BinaryAttachment, + field, + CbBinaryAttachment(IoHash(sequentialBytes.asReadOnlyBuffer())), + CbBinaryAttachment(IoHash.ZERO), + None + ) + + // Test CbField(None) as BinaryAttachment + val defaultField = CbField() + testField( + fieldType = CbFieldType.BinaryAttachment, + field = defaultField, + expectedError = TypeError, + expectedValue = CbBinaryAttachment(IoHash(sequentialBytes.asReadOnlyBuffer())), + defaultValue = CbBinaryAttachment(IoHash(sequentialBytes.asReadOnlyBuffer())), + ) + } + + @Test + fun iterateAttachmentTest() { + val hash = BlakeUtils.hash("hello world") + + val writer = CbWriter().apply { + beginObject() + writeString("TestString", "test") + writeBinaryAttachment("TestBinary", hash) + writeObjectAttachment("TestObject", hash) + + // inner object + beginObject("InnerObject") + writeString("TestString", "test") + writeBinaryAttachment("TestBinary", hash) + writeObjectAttachment("TestObject", hash) + endObject() + + // inner uniform array + beginArray("InnerArray", CbFieldType.BinaryAttachment) + writeBinaryAttachmentValue(hash) + writeBinaryAttachmentValue(hash) + endArray() + + // inner array + beginArray("InnerArray2") + writeStringValue("test") + writeBinaryAttachmentValue(hash) + endArray() + + endObject() + } + + val field = CbField(ByteBuffer.wrap(writer.toByteArray())) + + var count = 0 + field.asObject().iterateAttachments { count++ } + assertEquals(7, count) + } + + @Test + fun hashTest() { + val zeroBytes = ByteBuffer.allocate(20) + val sequentialBytes = byteBufferOf( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 + ) + + // Test CbField(Hash, Zero) + testField(CbFieldType.Hash, zeroBytes) + + // Test CbField(Hash, NonZero) + testField(CbFieldType.Hash, sequentialBytes, IoHash(sequentialBytes.asReadOnlyBuffer())) + + // Test CbField(None) as Hash + val defaultField = CbField() + testField( + fieldType = CbFieldType.Hash, + field = defaultField, + expectedError = TypeError, + expectedValue = IoHash(sequentialBytes.asReadOnlyBuffer()), + defaultValue = IoHash(sequentialBytes.asReadOnlyBuffer()), + ) + + // Test CbField(ObjectAttachment) as Hash + val field = CbField(sequentialBytes.asReadOnlyBuffer(), CbFieldType.ObjectAttachment) + testField(CbFieldType.Hash, field, IoHash(sequentialBytes.asReadOnlyBuffer())) + + // Test CbField(BinaryAttachment) as Hash + val binaryAttachmentField = CbField(sequentialBytes.asReadOnlyBuffer(), CbFieldType.BinaryAttachment) + testField(CbFieldType.Hash, binaryAttachmentField, IoHash(sequentialBytes.asReadOnlyBuffer())) + } + + @Test + fun uuidTest() { + // TODO 与C# GUID不同，需要确认兼容性 + val zeroBytes = byteBufferOf(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + val sequentialBytes = byteBufferOf(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + val sequentialGuid = UUID.fromString("03020100-0504-0706-0809-0a0b0c0d0e0f") + + // Test CbField(Uuid, Zero) + testField(CbFieldType.Uuid, zeroBytes, UUID(0, 0), sequentialGuid) + + // Test CbField(Uuid, NonZero) + testField(CbFieldType.Uuid, sequentialBytes, sequentialGuid, UUID(0, 0)) + + // Test CbField(None) as Uuid + val defaultField = CbField() + val expectedValue = UUID.randomUUID() + testField( + fieldType = CbFieldType.Uuid, + field = defaultField, + expectedError = TypeError, + expectedValue = expectedValue, + defaultValue = expectedValue + ) + } + + // ------------ test utils ------------------ + enum class EIntType(val value: Byte) { + None(0x00), + Int8(0x01), + Int16(0x02), + Int32(0x04), + Int64(0x08), + UInt8(0x10), + UInt16(0x20), + UInt32(0x40), + UInt64(0x80.toByte()), + + // Masks for positive values requiring the specified number of bits. + Pos64(UInt64.value), + Pos63(Pos64.value or Int64.value), + Pos32(Pos63.value or UInt32.value), + Pos31(Pos32.value or Int32.value), + Pos16(Pos31.value or UInt16.value), + Pos15(Pos16.value or Int16.value), + Pos8(Pos15.value or UInt8.value), + Pos7(Pos8.value or Int8.value), + + // Masks for negative values requiring the specified number of bits. + Neg63(Int64.value), + Neg31(Neg63.value or Int32.value), + Neg15(Neg31.value or Int16.value), + Neg7(Neg15.value or Int8.value) + } + + private fun testIntegerField(fieldType: CbFieldType, expectedMask: EIntType, magnitude: Long) { + val payload = ByteBuffer.allocate(9) + val negative = (fieldType.value and 1.toByte()) + VarULong.writeUnsigned(payload, magnitude - negative) + val defaultValue = 8L + val expectedValue = if (negative != 0.toByte()) -magnitude else magnitude + val field = CbField(payload, fieldType) + + var finalExpectedValue: Any? = if ((expectedMask.value and EIntType.Int8.value) != 0.toByte()) { + expectedValue.toByte() + } else { + defaultValue.toByte() + } + testField( + CbFieldType.IntegerNegative, + field, + finalExpectedValue, + defaultValue.toByte(), + if ((expectedMask.value and EIntType.Int8.value) != 0.toByte()) None else RangeError, + CbFieldAccessors.fromStruct( + defaultValue = 0, + isType = { x -> x.isInteger() }, + asTypeWithDefault = { x, y -> x.asInt8(y) }) + ) + + finalExpectedValue = (if ((expectedMask.value and EIntType.Int16.value) != 0.toByte()) { + expectedValue.toShort() + } else { + defaultValue.toShort() + }) + testField( + CbFieldType.IntegerNegative, + field, + finalExpectedValue, + defaultValue.toShort(), + if ((expectedMask.value and EIntType.Int16.value) != 0.toByte()) None else RangeError, + CbFieldAccessors.fromStruct( + defaultValue = 0, + isType = { x -> x.isInteger() }, + asTypeWithDefault = { x, y -> x.asInt16(y) }) + ) + + finalExpectedValue = if ((expectedMask.value and EIntType.Int32.value) != 0.toByte()) { + expectedValue.toInt() + } else { + defaultValue.toInt() + } + testField( + CbFieldType.IntegerNegative, + field, + finalExpectedValue, + defaultValue.toInt(), + if ((expectedMask.value and EIntType.Int32.value) != 0.toByte()) None else RangeError, + CbFieldAccessors.fromStruct(0, { x -> x.isInteger() }, { x, y -> x.asInt32(y) }) + ) + + testField( + CbFieldType.IntegerNegative, + field, + ((if ((expectedMask.value and EIntType.Int64.value) != 0.toByte()) expectedValue else defaultValue)), + defaultValue, + if ((expectedMask.value and EIntType.Int64.value) != 0.toByte()) None else RangeError, + CbFieldAccessors.fromStruct( + 0L, + isType = { x -> x.isInteger() }, + asTypeWithDefault = { x, y -> x.asInt64(y) }) + ) + + finalExpectedValue = if ((expectedMask.value and EIntType.UInt8.value) != 0.toByte()) { + expectedValue.toByte() + } else { + defaultValue.toByte() + } + testField( + CbFieldType.IntegerPositive, + field, + finalExpectedValue, + defaultValue.toByte(), + if ((expectedMask.value and EIntType.UInt8.value) != 0.toByte()) None else RangeError, + CbFieldAccessors.fromStruct(0, { x -> x.isInteger() }, { x, y -> x.asUInt8(y) }) + ) + + finalExpectedValue = if ((expectedMask.value and EIntType.UInt16.value) != 0.toByte()) { + expectedValue.toShort() + } else { + defaultValue.toShort() + } + testField( + CbFieldType.IntegerPositive, + field, + finalExpectedValue, + defaultValue.toShort(), + if ((expectedMask.value and EIntType.UInt16.value) != 0.toByte()) None else RangeError, + CbFieldAccessors.fromStruct(0, { x -> x.isInteger() }, { x, y -> x.asUInt16(y) }) + ) + + finalExpectedValue = if ((expectedMask.value and EIntType.UInt32.value) != 0.toByte()) { + expectedValue.toInt() + } else { + defaultValue.toInt() + } + testField( + CbFieldType.IntegerPositive, + field, + finalExpectedValue, + defaultValue.toInt(), + if ((expectedMask.value and EIntType.UInt32.value) != 0.toByte()) None else RangeError, + CbFieldAccessors.fromStruct(0, { x -> x.isInteger() }, { x, y -> x.asUInt32(y) }) + ) + + testField( + CbFieldType.IntegerPositive, + field, + ((if ((expectedMask.value and EIntType.UInt64.value) != 0.toByte()) expectedValue else defaultValue)), + defaultValue, + if ((expectedMask.value and EIntType.UInt64.value) != 0.toByte()) None else RangeError, + CbFieldAccessors.fromStruct(0L, { x -> x.isInteger() }, { x, y -> x.asUInt64(y) }) + ) + } + + private fun testIntArray(array: CbArray, expectedNum: Int, expectedPayloadSize: Int) { + assertEquals(array.getSize(), expectedPayloadSize + CbFieldType.SIZE_OF_CB_FIELD_TYPE) + assertEquals(array.count, expectedNum) + + var actualNum = 0 + for (field in array) { + ++actualNum + assertEquals(field.asInt32(), actualNum) + } + assertEquals(actualNum, expectedNum) + + actualNum = 0 + for (field in array.asField()) { + ++actualNum + assertEquals(field.asInt32(), actualNum) + } + assertEquals(actualNum, expectedNum) + } + + private fun testIntObject(obj: CbObject, expectedNum: Int, expectedPayloadSize: Int) { + assertEquals(obj.getSize(), expectedPayloadSize + CbFieldType.SIZE_OF_CB_FIELD_TYPE) + + var actualNum = 0 + for (field in obj) { + ++actualNum + assertNotEquals(field.name.length, 0) + assertEquals(field.asInt32(), actualNum) + } + assertEquals(expectedNum, actualNum) + } + + private fun testField( + fieldType: CbFieldType, + payload: ByteBuffer, + expectedValue: Any? = null, + defaultValue: Any? = null, + expectedError: CbField.CbFieldError = None, + accessors: CbFieldAccessors? = null, + ) { + val field = CbField(payload, fieldType) + assertEquals( + payload.remaining() + if (CbFieldUtils.hasFieldType(fieldType.value)) 0 else 1, + field.getSize() + ) + assertTrue(field.hasValue()) + assertFalse(field.hasError()) + assertEquals(None, field.error) + testField(fieldType, field, expectedValue, defaultValue, expectedError, accessors) + } + + private fun testField( + fieldType: CbFieldType, + field: CbField, + expectedValue: Any? = null, + defaultValue: Any? = null, + expectedError: CbField.CbFieldError = None, + accessors: CbFieldAccessors? = null + ) { + val actualAccessors = accessors ?: typeAccessors[fieldType]!! + val actualExpectedValue = expectedValue ?: actualAccessors.defaultValue + val actualDefaultValue = defaultValue ?: actualAccessors.defaultValue + + assertEquals(actualAccessors.isType(field), expectedError != TypeError) + if (expectedError == None && !field.isBool()) { + assertFalse(field.asBool()) + assertTrue(field.hasError()) + assertEquals(field.error, TypeError) + } + + assertTrue(actualAccessors.asTypeWithDefault(field, actualDefaultValue) == actualExpectedValue) + assertEquals(field.hasError(), expectedError != None) + assertEquals(field.error, expectedError) + } + + private val typeAccessors = mapOf( + CbFieldType.Object to CbFieldAccessors( + CbObject.EMPTY, + { it.isObject() } + ) { x, y -> x.asObject() }, + CbFieldType.UniformObject to CbFieldAccessors( + CbObject.EMPTY, + { it.isObject() } + ) { x, y -> x.asObject() }, + CbFieldType.Array to CbFieldAccessors( + CbArray.EMPTY, + { it.isArray() } + ) { x, y -> x.asArray() }, + CbFieldType.UniformArray to CbFieldAccessors( + CbArray.EMPTY, + { it.isArray() } + ) { x, y -> x.asArray() }, + CbFieldType.Binary to CbFieldAccessors.fromStruct( + ByteBufferUtils.EMPTY, + { it.isBinary() }, + { x, y -> x.asBinary(y) } + ), + CbFieldType.String to CbFieldAccessors( + "", + { it.isString() } + ) { x, default -> x.asString(default as String) }, + CbFieldType.IntegerPositive to CbFieldAccessors.fromStruct( + 0, + { it.isInteger() }, + { x, y -> x.asUInt64(y) } + ), + CbFieldType.IntegerNegative to CbFieldAccessors.fromStruct( + 0, + { it.isInteger() }, + { x, y -> x.asInt64(y) } + ), + CbFieldType.Float32 to CbFieldAccessors.fromStruct( + 0.0f, + { it.isFloat() }, + { x, y -> x.asFloat(y) }, + { x, y -> x.compareTo(y) == 0 } + ), + CbFieldType.Float64 to CbFieldAccessors.fromStruct( + 0.0, + { it.isFloat() }, + { x, y -> x.asDouble(y) }, + { x, y -> x.compareTo(y) == 0 } + ), + CbFieldType.BoolTrue to CbFieldAccessors.fromStruct( + true, + { it.isBool() }, + { x, y -> x.asBool(y) } + ), + CbFieldType.BoolFalse to CbFieldAccessors.fromStruct( + false, + { it.isBool() }, + { x, y -> x.asBool(y) } + ), + CbFieldType.ObjectAttachment to CbFieldAccessors( + CbObjectAttachment(IoHash.ZERO), + { it.isObjectAttachment() } + ) { x, default -> x.asObjectAttachment(default as CbObjectAttachment) }, + CbFieldType.BinaryAttachment to CbFieldAccessors( + CbBinaryAttachment(IoHash.ZERO), + { it.isBinaryAttachment() } + ) { x, default -> x.asBinaryAttachment(default as CbBinaryAttachment) }, + CbFieldType.Hash to CbFieldAccessors( + IoHash.ZERO, + { it.isHash() } + ) { x, default -> x.asHash(default as IoHash) }, + CbFieldType.Uuid to CbFieldAccessors.fromStruct( + UUID.nameUUIDFromBytes(ByteArray(16)), + { it.isUuid() }, + { x, y -> x.asUuid(y) } + ), +// CbFieldType.DateTime to CbFieldAccessors.fromStruct( +// DateTime(0, DateTimeZone.UTC), +// { it.isDateTime() }, +// { x, y -> x.asDateTime(y) } +// ), +// CbFieldType.TimeSpan to CbFieldAccessors.fromStruct( +// { it.isTimeSpan() }, +// { x, y -> x.asTimeSpan(y) } +// ) + ) + + class CbFieldAccessors( + var defaultValue: Any?, + var isType: (CbField) -> Boolean, + var asTypeWithDefault: (CbField, Any?) -> Any?, + ) { + var comparer: (Any?, Any?) -> Boolean = { x, y -> x?.equals(y) == true } + + companion object { + inline fun fromStruct( + defaultValue: Any? = T::class.java.newInstance(), + noinline isType: (CbField) -> Boolean, + crossinline asTypeWithDefault: (CbField, T) -> Any?, + crossinline comparer: (T, T) -> Boolean = { x, y -> x == y }, + ): CbFieldAccessors { + return CbFieldAccessors( + defaultValue, + isType, + { field, y -> asTypeWithDefault(field, y as T) } + ).apply { this.comparer = { x, y -> comparer(x as T, y as T) } } + } + } + } +} diff --git a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterTest.kt b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterTest.kt new file mode 100644 index 0000000000..ad88114776 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/CbWriterTest.kt @@ -0,0 +1,182 @@ +package com.tencent.bkrepo.ddc.serialization + +import com.tencent.bkrepo.ddc.utils.beginUniformArray +import com.tencent.bkrepo.ddc.utils.writeBinaryReference +import com.tencent.bkrepo.ddc.utils.writeInteger +import com.tencent.bkrepo.ddc.utils.writeIntegerValue +import com.tencent.bkrepo.ddc.utils.writeNullValue +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.nio.ByteBuffer +import kotlin.experimental.or + +class CbWriterTest { + @Test + fun emptyObject() { + val writer = CbWriter() + writer.beginObject() + writer.endObject() + + val data = writer.toByteArray() + assertEquals(2, data.size) + assertEquals(data[0], CbFieldType.Object.value) + assertEquals(data[1], 0.toByte()) + } + + @Test + fun emptyArray() { + val writer = CbWriter() + writer.beginArray() + writer.endArray() + + val data = writer.toByteArray() + assertEquals(3, data.size) + assertEquals(data[0], CbFieldType.Array.value) + assertEquals(data[1], 1.toByte()) + assertEquals(data[2], 0.toByte()) + } + + @Test + fun objectTest() { + val writer = CbWriter() + writer.beginObject() + writer.writeInteger("a", 1) + writer.writeInteger("b", 2) + writer.writeInteger("c", 3) + writer.endObject() + + val data = writer.toByteArray() + assertEquals(14, data.size) + assertEquals(data[0], CbFieldType.Object.value) + assertEquals(data[1], 12.toByte()) // Length + + assertEquals(data[2], CbFieldType.IntegerPositive.value or CbFieldType.HasFieldName.value) + assertEquals(data[3], 1.toByte()) + assertEquals(data[4], 'a'.toByte()) + assertEquals(data[5], 1.toByte()) + + assertEquals(data[6], CbFieldType.IntegerPositive.value or CbFieldType.HasFieldName.value) + assertEquals(data[7], 1.toByte()) + assertEquals(data[8], 'b'.toByte()) + assertEquals(data[9], 2.toByte()) + + assertEquals(data[10], CbFieldType.IntegerPositive.value or CbFieldType.HasFieldName.value) + assertEquals(data[11], 1.toByte()) + assertEquals(data[12], 'c'.toByte()) + assertEquals(data[13], 3.toByte()) + } + + @Test + fun arrayTest() { + val writer = CbWriter() + writer.beginArray() + writer.writeIntegerValue(1) + writer.writeIntegerValue(2) + writer.writeIntegerValue(3) + writer.endArray() + + val data = writer.toByteArray() + assertEquals(9, data.size) + assertEquals(data[0], CbFieldType.Array.value) + assertEquals(data[1], 7.toByte()) // Length + assertEquals(data[2], 3.toByte()) // Item count + assertEquals(data[3], CbFieldType.IntegerPositive.value) + assertEquals(data[4], 1.toByte()) + assertEquals(data[5], CbFieldType.IntegerPositive.value) + assertEquals(data[6], 2.toByte()) + assertEquals(data[7], CbFieldType.IntegerPositive.value) + assertEquals(data[8], 3.toByte()) + } + + @Test + fun uniformArrayTest() { + val writer = CbWriter() + writer.beginUniformArray(CbFieldType.IntegerPositive) + writer.writeIntegerValue(1) + writer.writeIntegerValue(2) + writer.writeIntegerValue(3) + writer.endArray() + + val data = writer.toByteArray() + assertEquals(7, data.size) + assertEquals(data[0], CbFieldType.UniformArray.value) + assertEquals(data[1], 5.toByte()) // Length + assertEquals(data[2], 3.toByte()) // Item count + assertEquals(data[3], CbFieldType.IntegerPositive.value) + assertEquals(data[4], 1.toByte()) + assertEquals(data[5], 2.toByte()) + assertEquals(data[6], 3.toByte()) + } + + + @Test + fun nestedArray() { + val writer = CbWriter() + writer.beginObject() + writer.beginArray("a") + writer.writeIntegerValue(1) + writer.endArray() + writer.endObject() + + val data = writer.toByteArray() + assertEquals(9, data.size) + assertEquals(CbFieldType.Object.value, data[0]) + assertEquals(7, data[1].toLong()) // Length + + assertEquals((CbFieldType.Array.value or CbFieldType.HasFieldName.value), data[2]) + assertEquals(1, data[3]) // Name length + assertEquals('a'.toByte(), data[4]) // Name + assertEquals(3, data[5]) // Length + assertEquals(1, data[6]) // Item count + + assertEquals(CbFieldType.IntegerPositive.value, data[7]) + assertEquals(1, data[8]) + } + + @Test + fun rawData() { + val test = byteArrayOf(1, 2, 3, 4) + + val writer = CbWriter() + writer.beginObject() + writer.writeBinaryReference("a", ByteBuffer.wrap(test)) + writer.endObject() + + val data = writer.toByteArray() + assertEquals(10, data.size) + assertEquals(CbFieldType.Object.value, data[0]) + assertEquals(8, data[1]) // Length + + assertEquals((CbFieldType.Binary.value or CbFieldType.HasFieldName.value), data[2]) + assertEquals(1, data[3]) // Name length + assertEquals('a'.toByte(), data[4]) // Name + assertEquals(4, data[5]) // Length + + assertEquals(1, data[6]) + assertEquals(2, data[7]) + assertEquals(3, data[8]) + assertEquals(4, data[9]) + } + + @Test + fun uniformArrayOfArrays() { + val writer = CbWriter() + writer.beginUniformArray(CbFieldType.Array) + writer.beginArray() + writer.writeNullValue() + writer.endArray() + writer.endArray() + + val data = writer.toByteArray() + assertEquals(7, data.size) + assertEquals(CbFieldType.UniformArray.value, data[0]) + assertEquals(5, data[1]) // Length + assertEquals(1, data[2]) // Item Count + + assertEquals(CbFieldType.Array.value, data[3]) + assertEquals(2, data[4]) // Length + assertEquals(1, data[5]) // Item Count + + assertEquals(CbFieldType.Null.value, data[6]) + } +} diff --git a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/VarULongTest.kt b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/VarULongTest.kt new file mode 100644 index 0000000000..ca10d08542 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/serialization/VarULongTest.kt @@ -0,0 +1,128 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.serialization + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.nio.ByteBuffer + +class VarULongTest { + @Test + fun writeUnsigned() { + val buffer = ByteBuffer.allocate(9) + + // test 127 + var num = 127L + VarULong.writeUnsigned(buffer, num) + assertEquals(1, buffer.position()) + buffer.flip() + assertEquals(1, buffer.remaining()) + assertEquals(0b0111_1111.toByte(), buffer.get(0)) + assertEquals(num, VarULong.readUnsigned(buffer)) + + // test 128 + num = 128L + buffer.clear() + VarULong.writeUnsigned(buffer, num) + buffer.flip() + assertEquals(0b1000_0000.toByte(), buffer.get(0)) + assertEquals(0b1000_0000.toByte(), buffer.get(1)) + assertEquals(num, VarULong.readUnsigned(buffer)) + + num = 129 + buffer.clear() + VarULong.writeUnsigned(buffer, num) + buffer.flip() + assertEquals(0b1000_0000.toByte(), buffer.get(0)) + assertEquals(0b1000_0001.toByte(), buffer.get(1)) + assertEquals(num, VarULong.readUnsigned(buffer)) + + num = 65537 + buffer.clear() + VarULong.writeUnsigned(buffer, num) + buffer.flip() + assertEquals(0b1100_0001.toByte(), buffer.get(0)) + assertEquals(0b0000_0000.toByte(), buffer.get(1)) + assertEquals(0b0000_0001.toByte(), buffer.get(2)) + assertEquals(num, VarULong.readUnsigned(buffer)) + + num = 1L shl 56 + buffer.clear() + VarULong.writeUnsigned(buffer, num) + buffer.flip() + assertEquals(0b1111_1111.toByte(), buffer.get(0)) + assertEquals(0b0000_0001.toByte(), buffer.get(1)) + assertEquals(0b0000_0000.toByte(), buffer.get(2)) + assertEquals(0b0000_0000.toByte(), buffer.get(3)) + assertEquals(0b0000_0000.toByte(), buffer.get(4)) + assertEquals(0b0000_0000.toByte(), buffer.get(5)) + assertEquals(0b0000_0000.toByte(), buffer.get(6)) + assertEquals(0b0000_0000.toByte(), buffer.get(7)) + assertEquals(0b0000_0000.toByte(), buffer.get(8)) + assertEquals(num, VarULong.readUnsigned(buffer)) + + // negative + num = -129 // 0x1111_1111{7} 0x0111_1111 + buffer.clear() + VarULong.writeUnsigned(buffer, num) + buffer.flip() + assertEquals(0b1111_1111.toByte(), buffer.get(0)) + assertEquals(0b1111_1111.toByte(), buffer.get(7)) + assertEquals(0b0111_1111.toByte(), buffer.get(8)) + assertEquals(num, VarULong.readUnsigned(buffer)) + + num = Long.MIN_VALUE // 0x1000_0000 0x0000_0000{8} + buffer.clear() + VarULong.writeUnsigned(buffer, num) + buffer.flip() + assertEquals(0b1111_1111.toByte(), buffer.get(0)) + assertEquals(0b1000_0000.toByte(), buffer.get(1)) + assertEquals(0b0000_0000.toByte(), buffer.get(8)) + assertEquals(num, VarULong.readUnsigned(buffer)) + } + + @Test + fun writeSigned() { + val buffer = ByteBuffer.allocate(9) + testSigned(buffer, 1) + testSigned(buffer, -1) + + testSigned(buffer, 129L) + testSigned(buffer, -129L) + + testSigned(buffer, Long.MAX_VALUE) + testSigned(buffer, Long.MIN_VALUE) + } + + private fun testSigned(buffer: ByteBuffer, num: Long) { + buffer.clear() + VarULong.writeSigned(buffer, num) + buffer.flip() + assertEquals(num, VarULong.readSigned(buffer)) + } +} diff --git a/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/utils/BlakeUtilsTest.kt b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/utils/BlakeUtilsTest.kt new file mode 100644 index 0000000000..45989ee839 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/test/kotlin/com/tencent/bkrepo/ddc/utils/BlakeUtilsTest.kt @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.utils + +import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import com.tencent.bkrepo.ddc.utils.BlakeUtils.toBlake3InputStream +import org.junit.jupiter.api.Test + +import org.junit.jupiter.api.Assertions.* +import java.io.ByteArrayInputStream +import java.nio.ByteBuffer + +class BlakeUtilsTest { + + @Test + fun hash() { + val content = "Hello world" + assertEquals("e7e6fb7d2869d109b62cdb1227208d4016cdaa0a", BlakeUtils.hash(content).hex()) + } + + @Test + fun hashBuffer() { + val buffer = ByteBuffer.allocate(11) + buffer.put("Hello ".toByteArray()) + val buffer2 = buffer.slice() + buffer2.put("world".toByteArray()) + val buffers = listOf( + buffer.flip() as ByteBuffer, + buffer2.flip() as ByteBuffer + ) + assertEquals("e7e6fb7d2869d109b62cdb1227208d4016cdaa0a", BlakeUtils.hash(buffers).hex()) + } + + @Test + fun hashInputStream() { + val blake3InputStream = ByteArrayInputStream("Hello world".toByteArray()).toBlake3InputStream() + blake3InputStream.reader().readText() + assertEquals("e7e6fb7d2869d109b62cdb1227208d4016cdaa0a", blake3InputStream.hash().hex()) + } +} diff --git a/src/backend/ddc/boot-ddc/build.gradle.kts b/src/backend/ddc/boot-ddc/build.gradle.kts new file mode 100644 index 0000000000..a8ff34940f --- /dev/null +++ b/src/backend/ddc/boot-ddc/build.gradle.kts @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +dependencies { + implementation(project(":ddc:biz-ddc")) +} diff --git a/src/backend/ddc/boot-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/DdcApplication.kt b/src/backend/ddc/boot-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/DdcApplication.kt new file mode 100644 index 0000000000..278be2f3ec --- /dev/null +++ b/src/backend/ddc/boot-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/DdcApplication.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc + +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.runApplication + +@SpringBootApplication +class DdcApplication + +fun main(args: Array) { + runApplication(*args) +} diff --git a/src/backend/ddc/boot-ddc/src/main/resources/bootstrap.yml b/src/backend/ddc/boot-ddc/src/main/resources/bootstrap.yml new file mode 100644 index 0000000000..8a4ce3c0e5 --- /dev/null +++ b/src/backend/ddc/boot-ddc/src/main/resources/bootstrap.yml @@ -0,0 +1,5 @@ +server.port: 25817 + +spring: + application: + name: ddc diff --git a/src/backend/settings.gradle.kts b/src/backend/settings.gradle.kts index cc674e1f06..1073f560d9 100644 --- a/src/backend/settings.gradle.kts +++ b/src/backend/settings.gradle.kts @@ -87,3 +87,4 @@ includeAll(":conan") includeAll(":fs") includeAll(":config") includeAll(":lfs") +includeAll(":ddc")