From f615cdf0b51284d2408c7609e2ea47b6e8ea695f Mon Sep 17 00:00:00 2001 From: liuliaozhong Date: Thu, 5 Dec 2024 16:44:50 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20=E6=96=87=E4=BB=B6=E5=9C=A8?= =?UTF-8?q?=E7=BA=BF=E9=A2=84=E8=A7=88=E6=94=AF=E6=8C=81=E6=9B=B4=E5=A4=9A?= =?UTF-8?q?=E7=9A=84=E6=96=87=E4=BB=B6=E7=B1=BB=E5=9E=8B=20#2765?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 文件预览后端服务基础实现，为页面提供接口服务 1.一个全新的preview服务，已实现文件的下载，可以下载制品库文件、远程文件. 2.office文件转换实现. 3.水印接口. 4.缓存与异常处理. 5.k8s部署 --- .../buildSrc/src/main/kotlin/Versions.kt | 4 + src/backend/preview/api-preview/build.gradle | 37 + .../preview/artifact/PreviewArtifactInfo.kt | 45 + .../bkrepo/preview/constant/Constants.kt | 39 + .../preview/constant/PreviewMessageCode.kt | 47 + .../bkrepo/preview/pojo/DownloadResult.kt | 48 + .../bkrepo/preview/pojo/FileAttribute.kt | 80 + .../tencent/bkrepo/preview/pojo/FileType.kt | 210 + .../bkrepo/preview/pojo/PreviewInfo.kt | 60 + .../bkrepo/preview/pojo/PreviewOptions.kt | 53 + .../tencent/bkrepo/preview/pojo/Watermark.kt | 59 + src/backend/preview/biz-preview/build.gradle | 43 + .../preview/utils/SimpleEncodingDetects.java | 4598 +++++++++++++++++ .../config/PreviewArtifactInfoResolver.kt | 51 + .../config/PreviewArtifactResourceWriter.kt | 115 + .../preview/config/PreviewLocalRepository.kt | 73 + .../preview/config/PreviewRemoteRepository.kt | 41 + .../config/PreviewVirtualRepository.kt | 41 + .../PreviewArtifactConfigurer.kt | 58 + .../config/configuration/PreviewConfig.kt | 298 ++ .../configuration/ResourceWriterConfigurer.kt | 55 + .../config/startup/PreviewStartupRunner.kt | 115 + .../controller/CommonResourceController.kt | 64 + .../controller/FilePreviewController.kt | 132 + .../exception/PreviewExceptionHandler.kt | 87 + .../exception/PreviewHandleException.kt | 45 + .../exception/PreviewInvalidException.kt | 43 + .../exception/PreviewNotFoundException.kt | 43 + .../preview/service/CommonResourceService.kt | 65 + .../preview/service/FileHandlerService.kt | 285 + .../bkrepo/preview/service/FilePreview.kt | 35 + .../preview/service/FilePreviewFactory.kt | 14 + .../preview/service/FileTransferService.kt | 149 + .../preview/service/OfficePluginManager.kt | 168 + .../preview/service/OfficeToPdfService.kt | 122 + .../service/impl/OfficeFilePreviewImpl.kt | 169 + .../service/impl/OtherFilePreviewImpl.kt | 78 + .../bkrepo/preview/utils/DownloadUtils.kt | 197 + .../bkrepo/preview/utils/EncodingDetects.kt | 76 + .../tencent/bkrepo/preview/utils/FileUtils.kt | 250 + .../tencent/bkrepo/preview/utils/FtpUtils.kt | 103 + .../bkrepo/preview/utils/JsonMapper.kt | 132 + .../tencent/bkrepo/preview/utils/SslUtils.kt | 72 + .../bkrepo/preview/utils/UrlEncoderUtils.kt | 97 + .../tencent/bkrepo/preview/utils/WebUtils.kt | 369 ++ .../resources/i18n/messages_en.properties | 36 + .../resources/i18n/messages_zh_CN.properties | 36 + .../resources/i18n/messages_zh_TW.properties | 36 + src/backend/preview/boot-preview/build.gradle | 34 + .../bkrepo/preview/PreviewApplication.kt | 45 + .../src/main/resources/bootstrap.yml | 2 + src/backend/preview/build.gradle | 30 + src/backend/settings.gradle.kts | 1 + .../bkrepo/templates/preview/configmap.yaml | 22 + .../bkrepo/templates/preview/deployment.yaml | 137 + .../bkrepo/templates/preview/service.yaml | 24 + .../kubernetes/charts/bkrepo/values.yaml | 40 + .../images/backend/preview.Dockerfile | 47 + 58 files changed, 9455 insertions(+) create mode 100644 src/backend/preview/api-preview/build.gradle create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/artifact/PreviewArtifactInfo.kt create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/constant/Constants.kt create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/constant/PreviewMessageCode.kt create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/DownloadResult.kt create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileAttribute.kt create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileType.kt create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/PreviewInfo.kt create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/PreviewOptions.kt create mode 100644 src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/Watermark.kt create mode 100644 src/backend/preview/biz-preview/build.gradle create mode 100644 src/backend/preview/biz-preview/src/main/java/com/tencent/bkrepo/preview/utils/SimpleEncodingDetects.java create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewArtifactInfoResolver.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewArtifactResourceWriter.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewLocalRepository.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewRemoteRepository.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewVirtualRepository.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewArtifactConfigurer.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/ResourceWriterConfigurer.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/startup/PreviewStartupRunner.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/CommonResourceController.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/FilePreviewController.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewExceptionHandler.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewHandleException.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewInvalidException.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewNotFoundException.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/CommonResourceService.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreview.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreviewFactory.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileTransferService.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficeToPdfService.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/OfficeFilePreviewImpl.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/OtherFilePreviewImpl.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/EncodingDetects.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FileUtils.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FtpUtils.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/JsonMapper.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/UrlEncoderUtils.kt create mode 100644 src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/WebUtils.kt create mode 100644 src/backend/preview/biz-preview/src/main/resources/i18n/messages_en.properties create mode 100644 src/backend/preview/biz-preview/src/main/resources/i18n/messages_zh_CN.properties create mode 100644 src/backend/preview/biz-preview/src/main/resources/i18n/messages_zh_TW.properties create mode 100644 src/backend/preview/boot-preview/build.gradle create mode 100644 src/backend/preview/boot-preview/src/main/kotlin/com/tencent/bkrepo/preview/PreviewApplication.kt create mode 100644 src/backend/preview/boot-preview/src/main/resources/bootstrap.yml create mode 100644 src/backend/preview/build.gradle create mode 100644 support-files/kubernetes/charts/bkrepo/templates/preview/configmap.yaml create mode 100644 support-files/kubernetes/charts/bkrepo/templates/preview/deployment.yaml create mode 100644 support-files/kubernetes/charts/bkrepo/templates/preview/service.yaml create mode 100644 support-files/kubernetes/images/backend/preview.Dockerfile diff --git a/src/backend/buildSrc/src/main/kotlin/Versions.kt b/src/backend/buildSrc/src/main/kotlin/Versions.kt index 7a7671d51c..459a0d6829 100644 --- a/src/backend/buildSrc/src/main/kotlin/Versions.kt +++ b/src/backend/buildSrc/src/main/kotlin/Versions.kt @@ -72,4 +72,8 @@ object Versions { const val Notice = "1.0.0" const val SpringCloudFunction = "3.2.11" const val Audit = "1.0.8" + const val JodConverter = "4.4.6" + const val Galimatias = "0.2.1" + const val CommonsNet = "3.9.0" + const val JuniversalCharDet = "1.0.3" } diff --git a/src/backend/preview/api-preview/build.gradle b/src/backend/preview/api-preview/build.gradle new file mode 100644 index 0000000000..ab55126519 --- /dev/null +++ b/src/backend/preview/api-preview/build.gradle @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +dependencies { + api(project(":common:common-api")) + api(project(":repository:api-repository")) + api(project(":auth:api-auth")) + compileOnly("org.springframework:spring-web") +} diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/artifact/PreviewArtifactInfo.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/artifact/PreviewArtifactInfo.kt new file mode 100644 index 0000000000..03489c8492 --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/artifact/PreviewArtifactInfo.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.artifact + +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo + +class PreviewArtifactInfo( + projectId: String, + repoName: String, + artifactUri: String +) : ArtifactInfo(projectId, repoName, artifactUri) { + companion object { + const val PREVIEW_COMMON_MAPPING_URI = "/api/file/preview/{projectId}/{repoName}/**" + const val PREVIEW_REMOTE_MAPPING_URI = "/api/file/preview" + } +} diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/constant/Constants.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/constant/Constants.kt new file mode 100644 index 0000000000..9701e02c52 --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/constant/Constants.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.constant + +const val BKREPO_PREFIX = "X-BKREPO-" + +// 节点详情 +const val PREVIEW_NODE_DETAIL = BKREPO_PREFIX + "PREVIEW_NODE_DETAIL" +// 临时文件保存路径 +const val PREVIEW_TMP_FILE_SAVE_PATH = BKREPO_PREFIX + "PREVIEW_TMP_FILE_SAVE_PATH" diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/constant/PreviewMessageCode.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/constant/PreviewMessageCode.kt new file mode 100644 index 0000000000..df170bae25 --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/constant/PreviewMessageCode.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.constant + +import com.tencent.bkrepo.common.api.message.MessageCode + +enum class PreviewMessageCode(private val key: String) : MessageCode { + PREVIEW_FILE_NOT_FOUND("preview.file.not-found"), + PREVIEW_NODE_NOT_FOUND("preview.node.not-found"), + PREVIEW_REPO_NOT_FOUND("preview.repo.not-found"), + PREVIEW_FIlE_HANDLE_ERROR("preview.file.handle.error"), + PREVIEW_PARAMETER_INVALID("preview.parameter.invalid") + ; + + override fun getBusinessCode() = ordinal + 1 + override fun getKey() = key + override fun getModuleCode() = 26 +} diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/DownloadResult.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/DownloadResult.kt new file mode 100644 index 0000000000..6f68e8cf1d --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/DownloadResult.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.pojo + +/** + * 下载结果 + */ +data class DownloadResult( + var code: Int = CODE_SUCCESS, + var filePath: String? = null, + var msg: String? = null, + var size: Long? = 0, + var md5: String? = null +) { + companion object { + const val CODE_SUCCESS = 0 + const val CODE_FAIL = 1 + } +} \ No newline at end of file diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileAttribute.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileAttribute.kt new file mode 100644 index 0000000000..1ebb7e1cfa --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileAttribute.kt @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("预览文件信息") +data class FileAttribute ( + // 存储类型：0-bkrepo文件，1-远程文件 + var storageType: Int = 0, + + // 项目id，bkrepo文件需要 + var projectId: String? = null, + + // 仓库名称，bkrepo文件需要 + var repoName: String? = null, + + // 文件全路径，bkrepo文件需要 + var artifactUri: String? = null, + + // 文件名称 + var fileName: String? = null, + + // 文件的完整下载地址，远程文件需要 + var url: String? = null, + + // 文件的md5 + val md5: String? = null, + + // baseUrl + var baseUrl: String? = null, + + // office预览类型（image / pdf) + val officePreviewType: String = "pdf", + + // 转换后的文件名 + var outFilePath: String? = null, + val originFilePath: String? = null, + val cacheListName: String? = null, + var convertFileName: String? = null, + + // xlsx是否转成html + var isHtmlView: Boolean = false, + + // 文件类型 + var type: FileType? = null, + + // 文件后缀 + var suffix: String? = null, +) \ No newline at end of file diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileType.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileType.kt new file mode 100644 index 0000000000..fa1cfb6197 --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileType.kt @@ -0,0 +1,210 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.pojo + +/** + * Content :文件类型，文本，office，压缩包等等 + */ +enum class FileType(val instanceName: String) { + PICTURE("pictureFilePreviewImpl"), + COMPRESS("compressFilePreviewImpl"), + OFFICE("officeFilePreviewImpl"), + SIMTEXT("simTextFilePreviewImpl"), + PDF("pdfFilePreviewImpl"), + CODE("codeFilePreviewImpl"), + OTHER("otherFilePreviewImpl"), + MEDIA("mediaFilePreviewImpl"), + MEDIACONVERT("mediaFilePreviewImpl"), + MARKDOWN("markdownFilePreviewImpl"), + XML("xmlFilePreviewImpl"), + CAD("cadFilePreviewImpl"), + TIFF("tiffFilePreviewImpl"), + OFD("ofdFilePreviewImpl"), + EML("emlFilePreviewImpl"), + ONLINE3D("online3DFilePreviewImpl"), + XMIND("xmindFilePreviewImpl"), + SVG("svgFilePreviewImpl"), + EPUB("epubFilePreviewImpl"), + BPMN("bpmnFilePreviewImpl"), + DCM("dcmFilePreviewImpl"), + DRAWIO("drawioFilePreviewImpl"); + + companion object { + private val OFFICE_TYPES = arrayOf( + "docx", + "wps", + "doc", + "docm", + "xls", + "xlsx", + "csv", + "xlsm", + "ppt", + "pptx", + "vsd", + "rtf", + "odt", + "wmf", + "emf", + "dps", + "et", + "ods", + "ots", + "tsv", + "odp", + "otp", + "sxi", + "ott", + "vsdx", + "fodt", + "fods", + "xltx", + "tga", + "psd", + "dotm", + "ett", + "xlt", + "xltm", + "wpt", + "dot", + "xlam", + "dotx", + "xla", + "pages", + "eps" + ) + private val PICTURE_TYPES = arrayOf("jpg", "jpeg", "png", "gif", "bmp", "ico", "jfif", "webp") + private val ARCHIVE_TYPES = arrayOf("rar", "zip", "jar", "7-zip", "tar", "gzip", "7z") + private val ONLINE3D_TYPES = arrayOf( + "obj", + "3ds", + "stl", + "ply", + "off", + "3dm", + "fbx", + "dae", + "wrl", + "3mf", + "ifc", + "glb", + "o3dv", + "gltf", + "stp", + "bim", + "fcstd", + "step", + "iges", + "brep" + ) + private val EML_TYPES = arrayOf("eml") + private val XMIND_TYPES = arrayOf("xmind") + private val EPUB_TYPES = arrayOf("epub") + private val DCM_TYPES = arrayOf("dcm") + private val DRAWIO_TYPES = arrayOf("drawio") + private val XML_TYPES = arrayOf("xml", "xbrl") + private val TIFF_TYPES = arrayOf("tif", "tiff") + private val OFD_TYPES = arrayOf("ofd") + private val SVG_TYPES = arrayOf("svg") + private val CAD_TYPES = + arrayOf("dwg", "dxf", "dwf", "iges", "igs", "dwt", "dng", "ifc", "dwfx", "stl", "cf2", "plt") + private val CODES = arrayOf( + "java", + "c", + "php", + "go", + "python", + "py", + "js", + "html", + "ftl", + "css", + "lua", + "sh", + "rb", + "yaml", + "yml", + "json", + "h", + "cpp", + "cs", + "aspx", + "jsp", + "sql" + ) + + private val FILE_TYPE_MAPPER = mutableMapOf() + + init { + OFFICE_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = OFFICE } + PICTURE_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = PICTURE } + ARCHIVE_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = COMPRESS } + TIFF_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = TIFF } + CODES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = CODE } + OFD_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = OFD } + CAD_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = CAD } + SVG_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = SVG } + EPUB_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = EPUB } + EML_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = EML } + XMIND_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = XMIND } + ONLINE3D_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = ONLINE3D } + DCM_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = DCM } + DRAWIO_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = DRAWIO } + XML_TYPES.forEach { fileType -> FILE_TYPE_MAPPER[fileType] = XML } + FILE_TYPE_MAPPER["md"] = MARKDOWN + FILE_TYPE_MAPPER["pdf"] = PDF + FILE_TYPE_MAPPER["bpmn"] = BPMN + } + + private fun to(fileType: String): FileType { + return FILE_TYPE_MAPPER[fileType] ?: OTHER + } + + /** + * 查看文件类型(防止参数中存在.点号或者其他特殊字符，所以先抽取文件名，然后再获取文件类型) + * + * @param url url + * @return 文件类型 + */ + fun typeFromUrl(url: String): FileType { + val nonPramStr = url.substring(0, url.indexOf("?").takeIf { it != -1 } ?: url.length) + val fileName = nonPramStr.substring(nonPramStr.lastIndexOf("/") + 1) + return typeFromFileName(fileName) + } + + fun typeFromFileName(fileName: String): FileType { + val fileType = fileName.substring(fileName.lastIndexOf(".") + 1) + val lowerCaseFileType = fileType.lowercase() + return to(lowerCaseFileType) + } + } +} diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/PreviewInfo.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/PreviewInfo.kt new file mode 100644 index 0000000000..4bc1b8ab4d --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/PreviewInfo.kt @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +/** + * 预览信息 + */ +@ApiModel("预览信息") +data class PreviewInfo ( + @ApiModelProperty("原始的文件属性") + val file: FileAttribute? = null, + @ApiModelProperty("图片集合") + val imgUrls: List? = null, + @ApiModelProperty("pdf的预览地址") + var pdfUrl: String? = null, + @ApiModelProperty("预览地址") + var url: String? = null, + @ApiModelProperty("文件名称") + var fileName: String? = null, + @ApiModelProperty("错误信息") + var msg: String? = null, + + @ApiModelProperty("文件类型") + var fileType: String? = null, + // 文件展示模板，如officeweb、ppt等 + @ApiModelProperty("文件展示模板") + var fileTemplate: String? = null, +) \ No newline at end of file diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/PreviewOptions.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/PreviewOptions.kt new file mode 100644 index 0000000000..0de24e38bb --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/PreviewOptions.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("预览属性") +data class PreviewOptions( + @ApiModelProperty("是否禁止演示模式") + var pdfPresentationModeDisable: Boolean = true, + @ApiModelProperty("是否禁止打开文件") + var pdfOpenFileDisable: Boolean = true, + @ApiModelProperty("是否禁止打印转换生成的PDF文件") + var pdfPrintDisable: Boolean = true, + @ApiModelProperty("是否禁止下载转换生成的PDF文件") + var pdfDownloadDisable: Boolean = true, + @ApiModelProperty("是否禁止bookmarkFileConvertQueueTask") + var pdfBookmarkDisable: Boolean = true, + @ApiModelProperty("是否禁止签名") + var pdfDisableEditing: Boolean = true, + @ApiModelProperty("是否关闭 office 预览切换开关") + var switchDisabled: Boolean = true, +) \ No newline at end of file diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/Watermark.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/Watermark.kt new file mode 100644 index 0000000000..40ee13ff25 --- /dev/null +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/Watermark.kt @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("水印信息") +data class Watermark( + @ApiModelProperty("水印内容") + var watermarkTxt: String? = null, + @ApiModelProperty("水印x轴间隔") + var watermarkXSpace: String? = null, + @ApiModelProperty("水印y轴间隔") + var watermarkYSpace: String? = null, + @ApiModelProperty("水印字体") + var watermarkFont: String? = null, + @ApiModelProperty("水印字体大小") + var watermarkFontsize: String? = null, + @ApiModelProperty("水印颜色") + var watermarkColor: String? = null, + @ApiModelProperty("水印透明度") + var watermarkAlpha: String? = null, + @ApiModelProperty("水印宽度") + var watermarkWidth: String? = null, + @ApiModelProperty("水印高度") + var watermarkHeight: String? = null, + @ApiModelProperty("水印倾斜度数") + var watermarkAngle: String? = null, +) \ No newline at end of file diff --git a/src/backend/preview/biz-preview/build.gradle b/src/backend/preview/biz-preview/build.gradle new file mode 100644 index 0000000000..d388aa14b1 --- /dev/null +++ b/src/backend/preview/biz-preview/build.gradle @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +dependencies { + api(project(":preview:api-preview")) + //api(project(":common:common-generic")) + api(project(":common:common-artifact:artifact-service")) + api("org.dom4j:dom4j:${Versions.Dom4j}") + // office转换 + api("org.jodconverter:jodconverter-local:${Versions.JodConverter}") + // url规范化 + api("io.mola.galimatias:galimatias:${Versions.Galimatias}") + api("commons-net:commons-net:${Versions.CommonsNet}") + //编码检测-JUniversalCharDet--> + api("com.googlecode.juniversalchardet:juniversalchardet:${Versions.JuniversalCharDet}") +} diff --git a/src/backend/preview/biz-preview/src/main/java/com/tencent/bkrepo/preview/utils/SimpleEncodingDetects.java b/src/backend/preview/biz-preview/src/main/java/com/tencent/bkrepo/preview/utils/SimpleEncodingDetects.java new file mode 100644 index 0000000000..44cb66e831 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/java/com/tencent/bkrepo/preview/utils/SimpleEncodingDetects.java @@ -0,0 +1,4598 @@ +package com.tencent.bkrepo.preview.utils; + +import java.io.*; +import java.net.URL; + +/** + * + * Copyright (C) <2009> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * EncodingDetect.java
+ * 自动获取文件的编码 + * @author Billows.Van + * @since Create on 2010-01-27 11:19:00 + * @version 1.0 + */ +public class SimpleEncodingDetects { + + /** + * 得到文件的编码 + * @param content 文件内容 + * @return 文件的编码 + */ + public static String getJavaEncode(byte[] content){ + BytesEncodingDetect s = new BytesEncodingDetect(); + String fileCode = BytesEncodingDetect.javaname[s.detectEncoding(content)]; + return fileCode; + } + + public static void readFile(String file, String code) { + BufferedReader fr; + try { + String myCode = code!=null&&!"".equals(code) ? code : "UTF8"; + InputStreamReader read = new InputStreamReader(new FileInputStream( + file), myCode); + + fr = new BufferedReader(read); + String line = null; + int flag=1; + // 读取每一行，如果结束了，line会为空 + while ((line = fr.readLine()) != null && line.trim().length() > 0) { + if(flag==1) { + line=line.substring(1);//去掉文件头 + flag++; + } + // 每一行创建一个Student对象，并存入数组中 + System.out.println(line); + } + fr.close(); + + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } +} + +class BytesEncodingDetect extends Encoding { + // Frequency tables to hold the GB, Big5, and EUC-TW character + // frequencies + int GBFreq[][]; + + int GBKFreq[][]; + + int Big5Freq[][]; + + int Big5PFreq[][]; + + int EUC_TWFreq[][]; + + int KRFreq[][]; + + int JPFreq[][]; + + // int UnicodeFreq[94][128]; + // public static String[] nicename; + // public static String[] codings; + public boolean debug; + + public BytesEncodingDetect() { + super(); + debug = false; + GBFreq = new int[94][94]; + GBKFreq = new int[126][191]; + Big5Freq = new int[94][158]; + Big5PFreq = new int[126][191]; + EUC_TWFreq = new int[94][94]; + KRFreq = new int[94][94]; + JPFreq = new int[94][94]; + // Initialize the Frequency Table for GB, GBK, Big5, EUC-TW, KR, JP + initialize_frequencies(); + } + + + /** + * Function : detectEncoding Aruguments: URL Returns : One of the encodings from the Encoding enumeration (GB2312, HZ, BIG5, + * EUC_TW, ASCII, or OTHER) Description: This function looks at the URL contents and assigns it a probability score for each + * encoding type. The encoding type with the highest probability is returned. + */ + public int detectEncoding(URL testurl) { + byte[] rawtext = new byte[10000]; + int bytesread = 0, byteoffset = 0; + int guess = OTHER; + InputStream chinesestream; + try { + chinesestream = testurl.openStream(); + while ((bytesread = chinesestream.read(rawtext, byteoffset, rawtext.length - byteoffset)) > 0) { + byteoffset += bytesread; + } + ; + chinesestream.close(); + guess = detectEncoding(rawtext); + } catch (Exception e) { + System.err.println("Error loading or using URL " + e.toString()); + guess = -1; + } + return guess; + } + + + /** + * Function : detectEncoding Aruguments: byte array Returns : One of the encodings from the Encoding enumeration (GB2312, HZ, + * BIG5, EUC_TW, ASCII, or OTHER) Description: This function looks at the byte array and assigns it a probability score for + * each encoding type. The encoding type with the highest probability is returned. + */ + public int detectEncoding(byte[] rawtext) { + int[] scores; + int index, maxscore = 0; + int encoding_guess = OTHER; + scores = new int[TOTALTYPES]; + // Assign Scores + scores[GB2312] = gb2312_probability(rawtext); + scores[GBK] = gbk_probability(rawtext); + scores[GB18030] = gb18030_probability(rawtext); + scores[HZ] = hz_probability(rawtext); + scores[BIG5] = big5_probability(rawtext); + scores[CNS11643] = euc_tw_probability(rawtext); + scores[ISO2022CN] = iso_2022_cn_probability(rawtext); + scores[UTF8] = utf8_probability(rawtext); + scores[UNICODE] = utf16_probability(rawtext); + scores[EUC_KR] = euc_kr_probability(rawtext); + scores[CP949] = cp949_probability(rawtext); + scores[JOHAB] = 0; + scores[ISO2022KR] = iso_2022_kr_probability(rawtext); + scores[ASCII] = ascii_probability(rawtext); + scores[SJIS] = sjis_probability(rawtext); + scores[EUC_JP] = euc_jp_probability(rawtext); + scores[ISO2022JP] = iso_2022_jp_probability(rawtext); + scores[UNICODET] = 0; + scores[UNICODES] = 0; + scores[ISO2022CN_GB] = 0; + scores[ISO2022CN_CNS] = 0; + scores[OTHER] = 0; + // Tabulate Scores + for (index = 0; index < TOTALTYPES; index++) { + if (debug) + System.err.println("Encoding " + nicename[index] + " score " + scores[index]); + if (scores[index] > maxscore) { + encoding_guess = index; + maxscore = scores[index]; + } + } + // Return OTHER if nothing scored above 50 + if (maxscore <= 50) { + encoding_guess = OTHER; + } + return encoding_guess; + } + + /* + * Function: gb2312_probability Argument: pointer to byte array Returns : number from 0 to 100 representing probability text + * in array uses GB-2312 encoding + */ + int gb2312_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, gbchars = 1; + long gbfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Stage 1: Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + // System.err.println(rawtext[i]); + if (rawtext[i] >= 0) { + // asciichars++; + } else { + dbchars++; + if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xF7 && (byte) 0xA1 <= rawtext[i + 1] + && rawtext[i + 1] <= (byte) 0xFE) { + gbchars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0xA1; + column = rawtext[i + 1] + 256 - 0xA1; + if (GBFreq[row][column] != 0) { + gbfreq += GBFreq[row][column]; + } else if (15 <= row && row < 55) { + // In GB high-freq character range + gbfreq += 200; + } + } + i++; + } + } + rangeval = 50 * ((float) gbchars / (float) dbchars); + freqval = 50 * ((float) gbfreq / (float) totalfreq); + return (int) (rangeval + freqval); + } + + /* + * Function: gbk_probability Argument: pointer to byte array Returns : number from 0 to 100 representing probability text in + * array uses GBK encoding + */ + int gbk_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, gbchars = 1; + long gbfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Stage 1: Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + // System.err.println(rawtext[i]); + if (rawtext[i] >= 0) { + // asciichars++; + } else { + dbchars++; + if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xF7 && // Original GB range + (byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE) { + gbchars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0xA1; + column = rawtext[i + 1] + 256 - 0xA1; + // System.out.println("original row " + row + " column " + column); + if (GBFreq[row][column] != 0) { + gbfreq += GBFreq[row][column]; + } else if (15 <= row && row < 55) { + gbfreq += 200; + } + } else if ((byte) 0x81 <= rawtext[i] + && rawtext[i] <= (byte) 0xFE + && // Extended GB range + (((byte) 0x80 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE) || ((byte) 0x40 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x7E))) { + gbchars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0x81; + if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) { + column = rawtext[i + 1] - 0x40; + } else { + column = rawtext[i + 1] + 256 - 0x40; + } + // System.out.println("extended row " + row + " column " + column + " rawtext[i] " + rawtext[i]); + if (GBKFreq[row][column] != 0) { + gbfreq += GBKFreq[row][column]; + } + } + i++; + } + } + rangeval = 50 * ((float) gbchars / (float) dbchars); + freqval = 50 * ((float) gbfreq / (float) totalfreq); + // For regular GB files, this would give the same score, so I handicap it slightly + return (int) (rangeval + freqval) - 1; + } + + /* + * Function: gb18030_probability Argument: pointer to byte array Returns : number from 0 to 100 representing probability text + * in array uses GBK encoding + */ + int gb18030_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, gbchars = 1; + long gbfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Stage 1: Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + // System.err.println(rawtext[i]); + if (rawtext[i] >= 0) { + // asciichars++; + } else { + dbchars++; + if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xF7 && // Original GB range + i + 1 < rawtextlen && (byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE) { + gbchars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0xA1; + column = rawtext[i + 1] + 256 - 0xA1; + // System.out.println("original row " + row + " column " + column); + if (GBFreq[row][column] != 0) { + gbfreq += GBFreq[row][column]; + } else if (15 <= row && row < 55) { + gbfreq += 200; + } + } else if ((byte) 0x81 <= rawtext[i] && rawtext[i] <= (byte) 0xFE + && // Extended GB range + i + 1 < rawtextlen + && (((byte) 0x80 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE) || ((byte) 0x40 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x7E))) { + gbchars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0x81; + if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) { + column = rawtext[i + 1] - 0x40; + } else { + column = rawtext[i + 1] + 256 - 0x40; + } + // System.out.println("extended row " + row + " column " + column + " rawtext[i] " + rawtext[i]); + if (GBKFreq[row][column] != 0) { + gbfreq += GBKFreq[row][column]; + } + } else if ((byte) 0x81 <= rawtext[i] + && rawtext[i] <= (byte) 0xFE + && // Extended GB range + i + 3 < rawtextlen && (byte) 0x30 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x39 + && (byte) 0x81 <= rawtext[i + 2] && rawtext[i + 2] <= (byte) 0xFE && (byte) 0x30 <= rawtext[i + 3] + && rawtext[i + 3] <= (byte) 0x39) { + gbchars++; + /* + * totalfreq += 500; row = rawtext[i] + 256 - 0x81; if (0x40 <= rawtext[i+1] && rawtext[i+1] <= 0x7E) { column = + * rawtext[i+1] - 0x40; } else { column = rawtext[i+1] + 256 - 0x40; } //System.out.println("extended row " + row + " + * column " + column + " rawtext[i] " + rawtext[i]); if (GBKFreq[row][column] != 0) { gbfreq += GBKFreq[row][column]; } + */ + } + i++; + } + } + rangeval = 50 * ((float) gbchars / (float) dbchars); + freqval = 50 * ((float) gbfreq / (float) totalfreq); + // For regular GB files, this would give the same score, so I handicap it slightly + return (int) (rangeval + freqval) - 1; + } + + /* + * Function: hz_probability Argument: byte array Returns : number from 0 to 100 representing probability text in array uses HZ + * encoding + */ + int hz_probability(byte[] rawtext) { + int i, rawtextlen; + int hzchars = 0, dbchars = 1; + long hzfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int hzstart = 0, hzend = 0; + int row, column; + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen; i++) { + if (rawtext[i] == '~' && i < rawtextlen - 1) { + if (rawtext[i + 1] == '{') { + hzstart++; + i += 2; + while (i < rawtextlen - 1) { + if (rawtext[i] == 0x0A || rawtext[i] == 0x0D) { + break; + } else if (rawtext[i] == '~' && rawtext[i + 1] == '}') { + hzend++; + i++; + break; + } else if ((0x21 <= rawtext[i] && rawtext[i] <= 0x77) && (0x21 <= rawtext[i + 1] && rawtext[i + 1] <= 0x77)) { + hzchars += 2; + row = rawtext[i] - 0x21; + column = rawtext[i + 1] - 0x21; + totalfreq += 500; + if (GBFreq[row][column] != 0) { + hzfreq += GBFreq[row][column]; + } else if (15 <= row && row < 55) { + hzfreq += 200; + } + } else if ((0xA1 <= rawtext[i] && rawtext[i] <= 0xF7) && (0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= 0xF7)) { + hzchars += 2; + row = rawtext[i] + 256 - 0xA1; + column = rawtext[i + 1] + 256 - 0xA1; + totalfreq += 500; + if (GBFreq[row][column] != 0) { + hzfreq += GBFreq[row][column]; + } else if (15 <= row && row < 55) { + hzfreq += 200; + } + } + dbchars += 2; + i += 2; + } + } else if (rawtext[i + 1] == '}') { + hzend++; + i++; + } else if (rawtext[i + 1] == '~') { + i++; + } + } + } + if (hzstart > 4) { + rangeval = 50; + } else if (hzstart > 1) { + rangeval = 41; + } else if (hzstart > 0) { // Only 39 in case the sequence happened to occur + rangeval = 39; // in otherwise non-Hz text + } else { + rangeval = 0; + } + freqval = 50 * ((float) hzfreq / (float) totalfreq); + return (int) (rangeval + freqval); + } + + /** + * Function: big5_probability Argument: byte array Returns : number from 0 to 100 representing probability text in array uses + * Big5 encoding + */ + int big5_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, bfchars = 1; + float rangeval = 0, freqval = 0; + long bffreq = 0, totalfreq = 1; + int row, column; + // Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + if (rawtext[i] >= 0) { + // asciichars++; + } else { + dbchars++; + if ((byte) 0xA1 <= rawtext[i] + && rawtext[i] <= (byte) 0xF9 + && (((byte) 0x40 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x7E) || ((byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE))) { + bfchars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0xA1; + if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) { + column = rawtext[i + 1] - 0x40; + } else { + column = rawtext[i + 1] + 256 - 0x61; + } + if (Big5Freq[row][column] != 0) { + bffreq += Big5Freq[row][column]; + } else if (3 <= row && row <= 37) { + bffreq += 200; + } + } + i++; + } + } + rangeval = 50 * ((float) bfchars / (float) dbchars); + freqval = 50 * ((float) bffreq / (float) totalfreq); + return (int) (rangeval + freqval); + } + + /* + * Function: big5plus_probability Argument: pointer to unsigned char array Returns : number from 0 to 100 representing + * probability text in array uses Big5+ encoding + */ + int big5plus_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, bfchars = 1; + long bffreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Stage 1: Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + // System.err.println(rawtext[i]); + if (rawtext[i] >= 128) { + // asciichars++; + } else { + dbchars++; + if (0xA1 <= rawtext[i] && rawtext[i] <= 0xF9 && // Original Big5 range + ((0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) || (0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= 0xFE))) { + bfchars++; + totalfreq += 500; + row = rawtext[i] - 0xA1; + if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) { + column = rawtext[i + 1] - 0x40; + } else { + column = rawtext[i + 1] - 0x61; + } + // System.out.println("original row " + row + " column " + column); + if (Big5Freq[row][column] != 0) { + bffreq += Big5Freq[row][column]; + } else if (3 <= row && row < 37) { + bffreq += 200; + } + } else if (0x81 <= rawtext[i] && rawtext[i] <= 0xFE && // Extended Big5 range + ((0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) || (0x80 <= rawtext[i + 1] && rawtext[i + 1] <= 0xFE))) { + bfchars++; + totalfreq += 500; + row = rawtext[i] - 0x81; + if (0x40 <= rawtext[i + 1] && rawtext[i + 1] <= 0x7E) { + column = rawtext[i + 1] - 0x40; + } else { + column = rawtext[i + 1] - 0x40; + } + // System.out.println("extended row " + row + " column " + column + " rawtext[i] " + rawtext[i]); + if (Big5PFreq[row][column] != 0) { + bffreq += Big5PFreq[row][column]; + } + } + i++; + } + } + rangeval = 50 * ((float) bfchars / (float) dbchars); + freqval = 50 * ((float) bffreq / (float) totalfreq); + // For regular Big5 files, this would give the same score, so I handicap it slightly + return (int) (rangeval + freqval) - 1; + } + + /* + * Function: euc_tw_probability Argument: byte array Returns : number from 0 to 100 representing probability text in array + * uses EUC-TW (CNS 11643) encoding + */ + int euc_tw_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, cnschars = 1; + long cnsfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Check to see if characters fit into acceptable ranges + // and have expected frequency of use + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + if (rawtext[i] >= 0) { // in ASCII range + // asciichars++; + } else { // high bit set + dbchars++; + if (i + 3 < rawtextlen && (byte) 0x8E == rawtext[i] && (byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xB0 + && (byte) 0xA1 <= rawtext[i + 2] && rawtext[i + 2] <= (byte) 0xFE && (byte) 0xA1 <= rawtext[i + 3] + && rawtext[i + 3] <= (byte) 0xFE) { // Planes 1 - 16 + cnschars++; + // System.out.println("plane 2 or above CNS char"); + // These are all less frequent chars so just ignore freq + i += 3; + } else if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xFE && // Plane 1 + (byte) 0xA1 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE) { + cnschars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0xA1; + column = rawtext[i + 1] + 256 - 0xA1; + if (EUC_TWFreq[row][column] != 0) { + cnsfreq += EUC_TWFreq[row][column]; + } else if (35 <= row && row <= 92) { + cnsfreq += 150; + } + i++; + } + } + } + rangeval = 50 * ((float) cnschars / (float) dbchars); + freqval = 50 * ((float) cnsfreq / (float) totalfreq); + return (int) (rangeval + freqval); + } + + /* + * Function: iso_2022_cn_probability Argument: byte array Returns : number from 0 to 100 representing probability text in + * array uses ISO 2022-CN encoding WORKS FOR BASIC CASES, BUT STILL NEEDS MORE WORK + */ + int iso_2022_cn_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, isochars = 1; + long isofreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Check to see if characters fit into acceptable ranges + // and have expected frequency of use + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + if (rawtext[i] == (byte) 0x1B && i + 3 < rawtextlen) { // Escape char ESC + if (rawtext[i + 1] == (byte) 0x24 && rawtext[i + 2] == 0x29 && rawtext[i + 3] == (byte) 0x41) { // GB Escape $ ) A + i += 4; + while (rawtext[i] != (byte) 0x1B) { + dbchars++; + if ((0x21 <= rawtext[i] && rawtext[i] <= 0x77) && (0x21 <= rawtext[i + 1] && rawtext[i + 1] <= 0x77)) { + isochars++; + row = rawtext[i] - 0x21; + column = rawtext[i + 1] - 0x21; + totalfreq += 500; + if (GBFreq[row][column] != 0) { + isofreq += GBFreq[row][column]; + } else if (15 <= row && row < 55) { + isofreq += 200; + } + i++; + } + i++; + } + } else if (i + 3 < rawtextlen && rawtext[i + 1] == (byte) 0x24 && rawtext[i + 2] == (byte) 0x29 + && rawtext[i + 3] == (byte) 0x47) { + // CNS Escape $ ) G + i += 4; + while (rawtext[i] != (byte) 0x1B) { + dbchars++; + if ((byte) 0x21 <= rawtext[i] && rawtext[i] <= (byte) 0x7E && (byte) 0x21 <= rawtext[i + 1] + && rawtext[i + 1] <= (byte) 0x7E) { + isochars++; + totalfreq += 500; + row = rawtext[i] - 0x21; + column = rawtext[i + 1] - 0x21; + if (EUC_TWFreq[row][column] != 0) { + isofreq += EUC_TWFreq[row][column]; + } else if (35 <= row && row <= 92) { + isofreq += 150; + } + i++; + } + i++; + } + } + if (rawtext[i] == (byte) 0x1B && i + 2 < rawtextlen && rawtext[i + 1] == (byte) 0x28 && rawtext[i + 2] == (byte) 0x42) { // ASCII: + // ESC + // ( B + i += 2; + } + } + } + rangeval = 50 * ((float) isochars / (float) dbchars); + freqval = 50 * ((float) isofreq / (float) totalfreq); + // System.out.println("isochars dbchars isofreq totalfreq " + isochars + " " + dbchars + " " + isofreq + " " + totalfreq + " + // " + rangeval + " " + freqval); + return (int) (rangeval + freqval); + // return 0; + } + + /* + * Function: utf8_probability Argument: byte array Returns : number from 0 to 100 representing probability text in array uses + * UTF-8 encoding of Unicode + */ + int utf8_probability(byte[] rawtext) { + int score = 0; + int i, rawtextlen = 0; + int goodbytes = 0, asciibytes = 0; + // Maybe also use UTF8 Byte Order Mark: EF BB BF + // Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen; i++) { + if ((rawtext[i] & (byte) 0x7F) == rawtext[i]) { // One byte + asciibytes++; + // Ignore ASCII, can throw off count + } else if (-64 <= rawtext[i] && rawtext[i] <= -33 && // Two bytes + i + 1 < rawtextlen && -128 <= rawtext[i + 1] && rawtext[i + 1] <= -65) { + goodbytes += 2; + i++; + } else if (-32 <= rawtext[i] && rawtext[i] <= -17 + && // Three bytes + i + 2 < rawtextlen && -128 <= rawtext[i + 1] && rawtext[i + 1] <= -65 && -128 <= rawtext[i + 2] + && rawtext[i + 2] <= -65) { + goodbytes += 3; + i += 2; + } + } + if (asciibytes == rawtextlen) { + return 0; + } + score = (int) (100 * ((float) goodbytes / (float) (rawtextlen - asciibytes))); + // System.out.println("rawtextlen " + rawtextlen + " goodbytes " + goodbytes + " asciibytes " + asciibytes + " score " + + // score); + // If not above 98, reduce to zero to prevent coincidental matches + // Allows for some (few) bad formed sequences + if (score > 98) { + return score; + } else if (score > 95 && goodbytes > 30) { + return score; + } else { + return 0; + } + } + + /* + * Function: utf16_probability Argument: byte array Returns : number from 0 to 100 representing probability text in array uses + * UTF-16 encoding of Unicode, guess based on BOM // NOT VERY GENERAL, NEEDS MUCH MORE WORK + */ + int utf16_probability(byte[] rawtext) { + // int score = 0; + // int i, rawtextlen = 0; + // int goodbytes = 0, asciibytes = 0; + if (rawtext.length > 1 && ((byte) 0xFE == rawtext[0] && (byte) 0xFF == rawtext[1]) || // Big-endian + ((byte) 0xFF == rawtext[0] && (byte) 0xFE == rawtext[1])) { // Little-endian + return 100; + } + return 0; + /* + * // Check to see if characters fit into acceptable ranges rawtextlen = rawtext.length; for (i = 0; i < rawtextlen; i++) { + * if ((rawtext[i] & (byte)0x7F) == rawtext[i]) { // One byte goodbytes += 1; asciibytes++; } else if ((rawtext[i] & + * (byte)0xDF) == rawtext[i]) { // Two bytes if (i+1 < rawtextlen && (rawtext[i+1] & (byte)0xBF) == rawtext[i+1]) { + * goodbytes += 2; i++; } } else if ((rawtext[i] & (byte)0xEF) == rawtext[i]) { // Three bytes if (i+2 < rawtextlen && + * (rawtext[i+1] & (byte)0xBF) == rawtext[i+1] && (rawtext[i+2] & (byte)0xBF) == rawtext[i+2]) { goodbytes += 3; i+=2; } } } + * + * score = (int)(100 * ((float)goodbytes/(float)rawtext.length)); // An all ASCII file is also a good UTF8 file, but I'd + * rather it // get identified as ASCII. Can delete following 3 lines otherwise if (goodbytes == asciibytes) { score = 0; } // + * If not above 90, reduce to zero to prevent coincidental matches if (score > 90) { return score; } else { return 0; } + */ + } + + /* + * Function: ascii_probability Argument: byte array Returns : number from 0 to 100 representing probability text in array uses + * all ASCII Description: Sees if array has any characters not in ASCII range, if so, score is reduced + */ + int ascii_probability(byte[] rawtext) { + int score = 75; + int i, rawtextlen; + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen; i++) { + if (rawtext[i] < 0) { + score = score - 5; + } else if (rawtext[i] == (byte) 0x1B) { // ESC (used by ISO 2022) + score = score - 5; + } + if (score <= 0) { + return 0; + } + } + return score; + } + + /* + * Function: euc_kr__probability Argument: pointer to byte array Returns : number from 0 to 100 representing probability text + * in array uses EUC-KR encoding + */ + int euc_kr_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, krchars = 1; + long krfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Stage 1: Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + // System.err.println(rawtext[i]); + if (rawtext[i] >= 0) { + // asciichars++; + } else { + dbchars++; + if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xFE && (byte) 0xA1 <= rawtext[i + 1] + && rawtext[i + 1] <= (byte) 0xFE) { + krchars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0xA1; + column = rawtext[i + 1] + 256 - 0xA1; + if (KRFreq[row][column] != 0) { + krfreq += KRFreq[row][column]; + } else if (15 <= row && row < 55) { + krfreq += 0; + } + } + i++; + } + } + rangeval = 50 * ((float) krchars / (float) dbchars); + freqval = 50 * ((float) krfreq / (float) totalfreq); + return (int) (rangeval + freqval); + } + + /* + * Function: cp949__probability Argument: pointer to byte array Returns : number from 0 to 100 representing probability text + * in array uses Cp949 encoding + */ + int cp949_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, krchars = 1; + long krfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Stage 1: Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + // System.err.println(rawtext[i]); + if (rawtext[i] >= 0) { + // asciichars++; + } else { + dbchars++; + if ((byte) 0x81 <= rawtext[i] + && rawtext[i] <= (byte) 0xFE + && ((byte) 0x41 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x5A || (byte) 0x61 <= rawtext[i + 1] + && rawtext[i + 1] <= (byte) 0x7A || (byte) 0x81 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFE)) { + krchars++; + totalfreq += 500; + if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xFE && (byte) 0xA1 <= rawtext[i + 1] + && rawtext[i + 1] <= (byte) 0xFE) { + row = rawtext[i] + 256 - 0xA1; + column = rawtext[i + 1] + 256 - 0xA1; + if (KRFreq[row][column] != 0) { + krfreq += KRFreq[row][column]; + } + } + } + i++; + } + } + rangeval = 50 * ((float) krchars / (float) dbchars); + freqval = 50 * ((float) krfreq / (float) totalfreq); + return (int) (rangeval + freqval); + } + + int iso_2022_kr_probability(byte[] rawtext) { + int i; + for (i = 0; i < rawtext.length; i++) { + if (i + 3 < rawtext.length && rawtext[i] == 0x1b && (char) rawtext[i + 1] == '$' && (char) rawtext[i + 2] == ')' + && (char) rawtext[i + 3] == 'C') { + return 100; + } + } + return 0; + } + + /* + * Function: euc_jp_probability Argument: pointer to byte array Returns : number from 0 to 100 representing probability text + * in array uses EUC-JP encoding + */ + int euc_jp_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, jpchars = 1; + long jpfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column; + // Stage 1: Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + // System.err.println(rawtext[i]); + if (rawtext[i] >= 0) { + // asciichars++; + } else { + dbchars++; + if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xFE && (byte) 0xA1 <= rawtext[i + 1] + && rawtext[i + 1] <= (byte) 0xFE) { + jpchars++; + totalfreq += 500; + row = rawtext[i] + 256 - 0xA1; + column = rawtext[i + 1] + 256 - 0xA1; + if (JPFreq[row][column] != 0) { + jpfreq += JPFreq[row][column]; + } else if (15 <= row && row < 55) { + jpfreq += 0; + } + } + i++; + } + } + rangeval = 50 * ((float) jpchars / (float) dbchars); + freqval = 50 * ((float) jpfreq / (float) totalfreq); + return (int) (rangeval + freqval); + } + + int iso_2022_jp_probability(byte[] rawtext) { + int i; + for (i = 0; i < rawtext.length; i++) { + if (i + 2 < rawtext.length && rawtext[i] == 0x1b && (char) rawtext[i + 1] == '$' && (char) rawtext[i + 2] == 'B') { + return 100; + } + } + return 0; + } + + /* + * Function: sjis_probability Argument: pointer to byte array Returns : number from 0 to 100 representing probability text in + * array uses Shift-JIS encoding + */ + int sjis_probability(byte[] rawtext) { + int i, rawtextlen = 0; + int dbchars = 1, jpchars = 1; + long jpfreq = 0, totalfreq = 1; + float rangeval = 0, freqval = 0; + int row, column, adjust; + // Stage 1: Check to see if characters fit into acceptable ranges + rawtextlen = rawtext.length; + for (i = 0; i < rawtextlen - 1; i++) { + // System.err.println(rawtext[i]); + if (rawtext[i] >= 0) { + // asciichars++; + } else { + dbchars++; + if (i + 1 < rawtext.length + && (((byte) 0x81 <= rawtext[i] && rawtext[i] <= (byte) 0x9F) || ((byte) 0xE0 <= rawtext[i] && rawtext[i] <= (byte) 0xEF)) + && (((byte) 0x40 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0x7E) || ((byte) 0x80 <= rawtext[i + 1] && rawtext[i + 1] <= (byte) 0xFC))) { + jpchars++; + totalfreq += 500; + row = rawtext[i] + 256; + column = rawtext[i + 1] + 256; + if (column < 0x9f) { + adjust = 1; + if (column > 0x7f) { + column -= 0x20; + } else { + column -= 0x19; + } + } else { + adjust = 0; + column -= 0x7e; + } + if (row < 0xa0) { + row = ((row - 0x70) << 1) - adjust; + } else { + row = ((row - 0xb0) << 1) - adjust; + } + row -= 0x20; + column = 0x20; + // System.out.println("original row " + row + " column " + column); + if (row < JPFreq.length && column < JPFreq[row].length && JPFreq[row][column] != 0) { + jpfreq += JPFreq[row][column]; + } + i++; + } else if ((byte) 0xA1 <= rawtext[i] && rawtext[i] <= (byte) 0xDF) { + // half-width katakana, convert to full-width + } + } + } + rangeval = 50 * ((float) jpchars / (float) dbchars); + freqval = 50 * ((float) jpfreq / (float) totalfreq); + // For regular GB files, this would give the same score, so I handicap it slightly + return (int) (rangeval + freqval) - 1; + } + + void initialize_frequencies() { + int i, j; + for (i = 0; i < 94; i++) { + for (j = 0; j < 94; j++) { + GBFreq[i][j] = 0; + } + } + for (i = 0; i < 126; i++) { + for (j = 0; j < 191; j++) { + GBKFreq[i][j] = 0; + } + } + for (i = 0; i < 94; i++) { + for (j = 0; j < 158; j++) { + Big5Freq[i][j] = 0; + } + } + for (i = 0; i < 126; i++) { + for (j = 0; j < 191; j++) { + Big5PFreq[i][j] = 0; + } + } + for (i = 0; i < 94; i++) { + for (j = 0; j < 94; j++) { + EUC_TWFreq[i][j] = 0; + } + } + for (i = 0; i < 94; i++) { + for (j = 0; j < 94; j++) { + JPFreq[i][j] = 0; + } + } + GBFreq[20][35] = 599; + GBFreq[49][26] = 598; + GBFreq[41][38] = 597; + GBFreq[17][26] = 596; + GBFreq[32][42] = 595; + GBFreq[39][42] = 594; + GBFreq[45][49] = 593; + GBFreq[51][57] = 592; + GBFreq[50][47] = 591; + GBFreq[42][90] = 590; + GBFreq[52][65] = 589; + GBFreq[53][47] = 588; + GBFreq[19][82] = 587; + GBFreq[31][19] = 586; + GBFreq[40][46] = 585; + GBFreq[24][89] = 584; + GBFreq[23][85] = 583; + GBFreq[20][28] = 582; + GBFreq[42][20] = 581; + GBFreq[34][38] = 580; + GBFreq[45][9] = 579; + GBFreq[54][50] = 578; + GBFreq[25][44] = 577; + GBFreq[35][66] = 576; + GBFreq[20][55] = 575; + GBFreq[18][85] = 574; + GBFreq[20][31] = 573; + GBFreq[49][17] = 572; + GBFreq[41][16] = 571; + GBFreq[35][73] = 570; + GBFreq[20][34] = 569; + GBFreq[29][44] = 568; + GBFreq[35][38] = 567; + GBFreq[49][9] = 566; + GBFreq[46][33] = 565; + GBFreq[49][51] = 564; + GBFreq[40][89] = 563; + GBFreq[26][64] = 562; + GBFreq[54][51] = 561; + GBFreq[54][36] = 560; + GBFreq[39][4] = 559; + GBFreq[53][13] = 558; + GBFreq[24][92] = 557; + GBFreq[27][49] = 556; + GBFreq[48][6] = 555; + GBFreq[21][51] = 554; + GBFreq[30][40] = 553; + GBFreq[42][92] = 552; + GBFreq[31][78] = 551; + GBFreq[25][82] = 550; + GBFreq[47][0] = 549; + GBFreq[34][19] = 548; + GBFreq[47][35] = 547; + GBFreq[21][63] = 546; + GBFreq[43][75] = 545; + GBFreq[21][87] = 544; + GBFreq[35][59] = 543; + GBFreq[25][34] = 542; + GBFreq[21][27] = 541; + GBFreq[39][26] = 540; + GBFreq[34][26] = 539; + GBFreq[39][52] = 538; + GBFreq[50][57] = 537; + GBFreq[37][79] = 536; + GBFreq[26][24] = 535; + GBFreq[22][1] = 534; + GBFreq[18][40] = 533; + GBFreq[41][33] = 532; + GBFreq[53][26] = 531; + GBFreq[54][86] = 530; + GBFreq[20][16] = 529; + GBFreq[46][74] = 528; + GBFreq[30][19] = 527; + GBFreq[45][35] = 526; + GBFreq[45][61] = 525; + GBFreq[30][9] = 524; + GBFreq[41][53] = 523; + GBFreq[41][13] = 522; + GBFreq[50][34] = 521; + GBFreq[53][86] = 520; + GBFreq[47][47] = 519; + GBFreq[22][28] = 518; + GBFreq[50][53] = 517; + GBFreq[39][70] = 516; + GBFreq[38][15] = 515; + GBFreq[42][88] = 514; + GBFreq[16][29] = 513; + GBFreq[27][90] = 512; + GBFreq[29][12] = 511; + GBFreq[44][22] = 510; + GBFreq[34][69] = 509; + GBFreq[24][10] = 508; + GBFreq[44][11] = 507; + GBFreq[39][92] = 506; + GBFreq[49][48] = 505; + GBFreq[31][46] = 504; + GBFreq[19][50] = 503; + GBFreq[21][14] = 502; + GBFreq[32][28] = 501; + GBFreq[18][3] = 500; + GBFreq[53][9] = 499; + GBFreq[34][80] = 498; + GBFreq[48][88] = 497; + GBFreq[46][53] = 496; + GBFreq[22][53] = 495; + GBFreq[28][10] = 494; + GBFreq[44][65] = 493; + GBFreq[20][10] = 492; + GBFreq[40][76] = 491; + GBFreq[47][8] = 490; + GBFreq[50][74] = 489; + GBFreq[23][62] = 488; + GBFreq[49][65] = 487; + GBFreq[28][87] = 486; + GBFreq[15][48] = 485; + GBFreq[22][7] = 484; + GBFreq[19][42] = 483; + GBFreq[41][20] = 482; + GBFreq[26][55] = 481; + GBFreq[21][93] = 480; + GBFreq[31][76] = 479; + GBFreq[34][31] = 478; + GBFreq[20][66] = 477; + GBFreq[51][33] = 476; + GBFreq[34][86] = 475; + GBFreq[37][67] = 474; + GBFreq[53][53] = 473; + GBFreq[40][88] = 472; + GBFreq[39][10] = 471; + GBFreq[24][3] = 470; + GBFreq[27][25] = 469; + GBFreq[26][15] = 468; + GBFreq[21][88] = 467; + GBFreq[52][62] = 466; + GBFreq[46][81] = 465; + GBFreq[38][72] = 464; + GBFreq[17][30] = 463; + GBFreq[52][92] = 462; + GBFreq[34][90] = 461; + GBFreq[21][7] = 460; + GBFreq[36][13] = 459; + GBFreq[45][41] = 458; + GBFreq[32][5] = 457; + GBFreq[26][89] = 456; + GBFreq[23][87] = 455; + GBFreq[20][39] = 454; + GBFreq[27][23] = 453; + GBFreq[25][59] = 452; + GBFreq[49][20] = 451; + GBFreq[54][77] = 450; + GBFreq[27][67] = 449; + GBFreq[47][33] = 448; + GBFreq[41][17] = 447; + GBFreq[19][81] = 446; + GBFreq[16][66] = 445; + GBFreq[45][26] = 444; + GBFreq[49][81] = 443; + GBFreq[53][55] = 442; + GBFreq[16][26] = 441; + GBFreq[54][62] = 440; + GBFreq[20][70] = 439; + GBFreq[42][35] = 438; + GBFreq[20][57] = 437; + GBFreq[34][36] = 436; + GBFreq[46][63] = 435; + GBFreq[19][45] = 434; + GBFreq[21][10] = 433; + GBFreq[52][93] = 432; + GBFreq[25][2] = 431; + GBFreq[30][57] = 430; + GBFreq[41][24] = 429; + GBFreq[28][43] = 428; + GBFreq[45][86] = 427; + GBFreq[51][56] = 426; + GBFreq[37][28] = 425; + GBFreq[52][69] = 424; + GBFreq[43][92] = 423; + GBFreq[41][31] = 422; + GBFreq[37][87] = 421; + GBFreq[47][36] = 420; + GBFreq[16][16] = 419; + GBFreq[40][56] = 418; + GBFreq[24][55] = 417; + GBFreq[17][1] = 416; + GBFreq[35][57] = 415; + GBFreq[27][50] = 414; + GBFreq[26][14] = 413; + GBFreq[50][40] = 412; + GBFreq[39][19] = 411; + GBFreq[19][89] = 410; + GBFreq[29][91] = 409; + GBFreq[17][89] = 408; + GBFreq[39][74] = 407; + GBFreq[46][39] = 406; + GBFreq[40][28] = 405; + GBFreq[45][68] = 404; + GBFreq[43][10] = 403; + GBFreq[42][13] = 402; + GBFreq[44][81] = 401; + GBFreq[41][47] = 400; + GBFreq[48][58] = 399; + GBFreq[43][68] = 398; + GBFreq[16][79] = 397; + GBFreq[19][5] = 396; + GBFreq[54][59] = 395; + GBFreq[17][36] = 394; + GBFreq[18][0] = 393; + GBFreq[41][5] = 392; + GBFreq[41][72] = 391; + GBFreq[16][39] = 390; + GBFreq[54][0] = 389; + GBFreq[51][16] = 388; + GBFreq[29][36] = 387; + GBFreq[47][5] = 386; + GBFreq[47][51] = 385; + GBFreq[44][7] = 384; + GBFreq[35][30] = 383; + GBFreq[26][9] = 382; + GBFreq[16][7] = 381; + GBFreq[32][1] = 380; + GBFreq[33][76] = 379; + GBFreq[34][91] = 378; + GBFreq[52][36] = 377; + GBFreq[26][77] = 376; + GBFreq[35][48] = 375; + GBFreq[40][80] = 374; + GBFreq[41][92] = 373; + GBFreq[27][93] = 372; + GBFreq[15][17] = 371; + GBFreq[16][76] = 370; + GBFreq[51][12] = 369; + GBFreq[18][20] = 368; + GBFreq[15][54] = 367; + GBFreq[50][5] = 366; + GBFreq[33][22] = 365; + GBFreq[37][57] = 364; + GBFreq[28][47] = 363; + GBFreq[42][31] = 362; + GBFreq[18][2] = 361; + GBFreq[43][64] = 360; + GBFreq[23][47] = 359; + GBFreq[28][79] = 358; + GBFreq[25][45] = 357; + GBFreq[23][91] = 356; + GBFreq[22][19] = 355; + GBFreq[25][46] = 354; + GBFreq[22][36] = 353; + GBFreq[54][85] = 352; + GBFreq[46][20] = 351; + GBFreq[27][37] = 350; + GBFreq[26][81] = 349; + GBFreq[42][29] = 348; + GBFreq[31][90] = 347; + GBFreq[41][59] = 346; + GBFreq[24][65] = 345; + GBFreq[44][84] = 344; + GBFreq[24][90] = 343; + GBFreq[38][54] = 342; + GBFreq[28][70] = 341; + GBFreq[27][15] = 340; + GBFreq[28][80] = 339; + GBFreq[29][8] = 338; + GBFreq[45][80] = 337; + GBFreq[53][37] = 336; + GBFreq[28][65] = 335; + GBFreq[23][86] = 334; + GBFreq[39][45] = 333; + GBFreq[53][32] = 332; + GBFreq[38][68] = 331; + GBFreq[45][78] = 330; + GBFreq[43][7] = 329; + GBFreq[46][82] = 328; + GBFreq[27][38] = 327; + GBFreq[16][62] = 326; + GBFreq[24][17] = 325; + GBFreq[22][70] = 324; + GBFreq[52][28] = 323; + GBFreq[23][40] = 322; + GBFreq[28][50] = 321; + GBFreq[42][91] = 320; + GBFreq[47][76] = 319; + GBFreq[15][42] = 318; + GBFreq[43][55] = 317; + GBFreq[29][84] = 316; + GBFreq[44][90] = 315; + GBFreq[53][16] = 314; + GBFreq[22][93] = 313; + GBFreq[34][10] = 312; + GBFreq[32][53] = 311; + GBFreq[43][65] = 310; + GBFreq[28][7] = 309; + GBFreq[35][46] = 308; + GBFreq[21][39] = 307; + GBFreq[44][18] = 306; + GBFreq[40][10] = 305; + GBFreq[54][53] = 304; + GBFreq[38][74] = 303; + GBFreq[28][26] = 302; + GBFreq[15][13] = 301; + GBFreq[39][34] = 300; + GBFreq[39][46] = 299; + GBFreq[42][66] = 298; + GBFreq[33][58] = 297; + GBFreq[15][56] = 296; + GBFreq[18][51] = 295; + GBFreq[49][68] = 294; + GBFreq[30][37] = 293; + GBFreq[51][84] = 292; + GBFreq[51][9] = 291; + GBFreq[40][70] = 290; + GBFreq[41][84] = 289; + GBFreq[28][64] = 288; + GBFreq[32][88] = 287; + GBFreq[24][5] = 286; + GBFreq[53][23] = 285; + GBFreq[42][27] = 284; + GBFreq[22][38] = 283; + GBFreq[32][86] = 282; + GBFreq[34][30] = 281; + GBFreq[38][63] = 280; + GBFreq[24][59] = 279; + GBFreq[22][81] = 278; + GBFreq[32][11] = 277; + GBFreq[51][21] = 276; + GBFreq[54][41] = 275; + GBFreq[21][50] = 274; + GBFreq[23][89] = 273; + GBFreq[19][87] = 272; + GBFreq[26][7] = 271; + GBFreq[30][75] = 270; + GBFreq[43][84] = 269; + GBFreq[51][25] = 268; + GBFreq[16][67] = 267; + GBFreq[32][9] = 266; + GBFreq[48][51] = 265; + GBFreq[39][7] = 264; + GBFreq[44][88] = 263; + GBFreq[52][24] = 262; + GBFreq[23][34] = 261; + GBFreq[32][75] = 260; + GBFreq[19][10] = 259; + GBFreq[28][91] = 258; + GBFreq[32][83] = 257; + GBFreq[25][75] = 256; + GBFreq[53][45] = 255; + GBFreq[29][85] = 254; + GBFreq[53][59] = 253; + GBFreq[16][2] = 252; + GBFreq[19][78] = 251; + GBFreq[15][75] = 250; + GBFreq[51][42] = 249; + GBFreq[45][67] = 248; + GBFreq[15][74] = 247; + GBFreq[25][81] = 246; + GBFreq[37][62] = 245; + GBFreq[16][55] = 244; + GBFreq[18][38] = 243; + GBFreq[23][23] = 242; + GBFreq[38][30] = 241; + GBFreq[17][28] = 240; + GBFreq[44][73] = 239; + GBFreq[23][78] = 238; + GBFreq[40][77] = 237; + GBFreq[38][87] = 236; + GBFreq[27][19] = 235; + GBFreq[38][82] = 234; + GBFreq[37][22] = 233; + GBFreq[41][30] = 232; + GBFreq[54][9] = 231; + GBFreq[32][30] = 230; + GBFreq[30][52] = 229; + GBFreq[40][84] = 228; + GBFreq[53][57] = 227; + GBFreq[27][27] = 226; + GBFreq[38][64] = 225; + GBFreq[18][43] = 224; + GBFreq[23][69] = 223; + GBFreq[28][12] = 222; + GBFreq[50][78] = 221; + GBFreq[50][1] = 220; + GBFreq[26][88] = 219; + GBFreq[36][40] = 218; + GBFreq[33][89] = 217; + GBFreq[41][28] = 216; + GBFreq[31][77] = 215; + GBFreq[46][1] = 214; + GBFreq[47][19] = 213; + GBFreq[35][55] = 212; + GBFreq[41][21] = 211; + GBFreq[27][10] = 210; + GBFreq[32][77] = 209; + GBFreq[26][37] = 208; + GBFreq[20][33] = 207; + GBFreq[41][52] = 206; + GBFreq[32][18] = 205; + GBFreq[38][13] = 204; + GBFreq[20][18] = 203; + GBFreq[20][24] = 202; + GBFreq[45][19] = 201; + GBFreq[18][53] = 200; + /* + * GBFreq[39][0] = 199; GBFreq[40][71] = 198; GBFreq[41][27] = 197; GBFreq[15][69] = 196; GBFreq[42][10] = 195; + * GBFreq[31][89] = 194; GBFreq[51][28] = 193; GBFreq[41][22] = 192; GBFreq[40][43] = 191; GBFreq[38][6] = 190; + * GBFreq[37][11] = 189; GBFreq[39][60] = 188; GBFreq[48][47] = 187; GBFreq[46][80] = 186; GBFreq[52][49] = 185; + * GBFreq[50][48] = 184; GBFreq[25][1] = 183; GBFreq[52][29] = 182; GBFreq[24][66] = 181; GBFreq[23][35] = 180; + * GBFreq[49][72] = 179; GBFreq[47][45] = 178; GBFreq[45][14] = 177; GBFreq[51][70] = 176; GBFreq[22][30] = 175; + * GBFreq[49][83] = 174; GBFreq[26][79] = 173; GBFreq[27][41] = 172; GBFreq[51][81] = 171; GBFreq[41][54] = 170; + * GBFreq[20][4] = 169; GBFreq[29][60] = 168; GBFreq[20][27] = 167; GBFreq[50][15] = 166; GBFreq[41][6] = 165; + * GBFreq[35][34] = 164; GBFreq[44][87] = 163; GBFreq[46][66] = 162; GBFreq[42][37] = 161; GBFreq[42][24] = 160; + * GBFreq[54][7] = 159; GBFreq[41][14] = 158; GBFreq[39][83] = 157; GBFreq[16][87] = 156; GBFreq[20][59] = 155; + * GBFreq[42][12] = 154; GBFreq[47][2] = 153; GBFreq[21][32] = 152; GBFreq[53][29] = 151; GBFreq[22][40] = 150; + * GBFreq[24][58] = 149; GBFreq[52][88] = 148; GBFreq[29][30] = 147; GBFreq[15][91] = 146; GBFreq[54][72] = 145; + * GBFreq[51][75] = 144; GBFreq[33][67] = 143; GBFreq[41][50] = 142; GBFreq[27][34] = 141; GBFreq[46][17] = 140; + * GBFreq[31][74] = 139; GBFreq[42][67] = 138; GBFreq[54][87] = 137; GBFreq[27][14] = 136; GBFreq[16][63] = 135; + * GBFreq[16][5] = 134; GBFreq[43][23] = 133; GBFreq[23][13] = 132; GBFreq[31][12] = 131; GBFreq[25][57] = 130; + * GBFreq[38][49] = 129; GBFreq[42][69] = 128; GBFreq[23][80] = 127; GBFreq[29][0] = 126; GBFreq[28][2] = 125; + * GBFreq[28][17] = 124; GBFreq[17][27] = 123; GBFreq[40][16] = 122; GBFreq[45][1] = 121; GBFreq[36][33] = 120; + * GBFreq[35][23] = 119; GBFreq[20][86] = 118; GBFreq[29][53] = 117; GBFreq[23][88] = 116; GBFreq[51][87] = 115; + * GBFreq[54][27] = 114; GBFreq[44][36] = 113; GBFreq[21][45] = 112; GBFreq[53][52] = 111; GBFreq[31][53] = 110; + * GBFreq[38][47] = 109; GBFreq[27][21] = 108; GBFreq[30][42] = 107; GBFreq[29][10] = 106; GBFreq[35][35] = 105; + * GBFreq[24][56] = 104; GBFreq[41][29] = 103; GBFreq[18][68] = 102; GBFreq[29][24] = 101; GBFreq[25][84] = 100; + * GBFreq[35][47] = 99; GBFreq[29][56] = 98; GBFreq[30][44] = 97; GBFreq[53][3] = 96; GBFreq[30][63] = 95; GBFreq[52][52] = + * 94; GBFreq[54][1] = 93; GBFreq[22][48] = 92; GBFreq[54][66] = 91; GBFreq[21][90] = 90; GBFreq[52][47] = 89; + * GBFreq[39][25] = 88; GBFreq[39][39] = 87; GBFreq[44][37] = 86; GBFreq[44][76] = 85; GBFreq[46][75] = 84; GBFreq[18][37] = + * 83; GBFreq[47][42] = 82; GBFreq[19][92] = 81; GBFreq[51][27] = 80; GBFreq[48][83] = 79; GBFreq[23][70] = 78; + * GBFreq[29][9] = 77; GBFreq[33][79] = 76; GBFreq[52][90] = 75; GBFreq[53][6] = 74; GBFreq[24][36] = 73; GBFreq[25][25] = + * 72; GBFreq[44][26] = 71; GBFreq[25][36] = 70; GBFreq[29][87] = 69; GBFreq[48][0] = 68; GBFreq[15][40] = 67; + * GBFreq[17][45] = 66; GBFreq[30][14] = 65; GBFreq[48][38] = 64; GBFreq[23][19] = 63; GBFreq[40][42] = 62; GBFreq[31][63] = + * 61; GBFreq[16][23] = 60; GBFreq[26][21] = 59; GBFreq[32][76] = 58; GBFreq[23][58] = 57; GBFreq[41][37] = 56; + * GBFreq[30][43] = 55; GBFreq[47][38] = 54; GBFreq[21][46] = 53; GBFreq[18][33] = 52; GBFreq[52][37] = 51; GBFreq[36][8] = + * 50; GBFreq[49][24] = 49; GBFreq[15][66] = 48; GBFreq[35][77] = 47; GBFreq[27][58] = 46; GBFreq[35][51] = 45; + * GBFreq[24][69] = 44; GBFreq[20][54] = 43; GBFreq[24][41] = 42; GBFreq[41][0] = 41; GBFreq[33][71] = 40; GBFreq[23][52] = + * 39; GBFreq[29][67] = 38; GBFreq[46][51] = 37; GBFreq[46][90] = 36; GBFreq[49][33] = 35; GBFreq[33][28] = 34; + * GBFreq[37][86] = 33; GBFreq[39][22] = 32; GBFreq[37][37] = 31; GBFreq[29][62] = 30; GBFreq[29][50] = 29; GBFreq[36][89] = + * 28; GBFreq[42][44] = 27; GBFreq[51][82] = 26; GBFreq[28][83] = 25; GBFreq[15][78] = 24; GBFreq[46][62] = 23; + * GBFreq[19][69] = 22; GBFreq[51][23] = 21; GBFreq[37][69] = 20; GBFreq[25][5] = 19; GBFreq[51][85] = 18; GBFreq[48][77] = + * 17; GBFreq[32][46] = 16; GBFreq[53][60] = 15; GBFreq[28][57] = 14; GBFreq[54][82] = 13; GBFreq[54][15] = 12; + * GBFreq[49][54] = 11; GBFreq[53][87] = 10; GBFreq[27][16] = 9; GBFreq[29][34] = 8; GBFreq[20][44] = 7; GBFreq[42][73] = 6; + * GBFreq[47][71] = 5; GBFreq[29][37] = 4; GBFreq[25][50] = 3; GBFreq[18][84] = 2; GBFreq[50][45] = 1; GBFreq[48][46] = 0; + */ + // GBFreq[43][89] = -1; GBFreq[54][68] = -2; + Big5Freq[9][89] = 600; + Big5Freq[11][15] = 599; + Big5Freq[3][66] = 598; + Big5Freq[6][121] = 597; + Big5Freq[3][0] = 596; + Big5Freq[5][82] = 595; + Big5Freq[3][42] = 594; + Big5Freq[5][34] = 593; + Big5Freq[3][8] = 592; + Big5Freq[3][6] = 591; + Big5Freq[3][67] = 590; + Big5Freq[7][139] = 589; + Big5Freq[23][137] = 588; + Big5Freq[12][46] = 587; + Big5Freq[4][8] = 586; + Big5Freq[4][41] = 585; + Big5Freq[18][47] = 584; + Big5Freq[12][114] = 583; + Big5Freq[6][1] = 582; + Big5Freq[22][60] = 581; + Big5Freq[5][46] = 580; + Big5Freq[11][79] = 579; + Big5Freq[3][23] = 578; + Big5Freq[7][114] = 577; + Big5Freq[29][102] = 576; + Big5Freq[19][14] = 575; + Big5Freq[4][133] = 574; + Big5Freq[3][29] = 573; + Big5Freq[4][109] = 572; + Big5Freq[14][127] = 571; + Big5Freq[5][48] = 570; + Big5Freq[13][104] = 569; + Big5Freq[3][132] = 568; + Big5Freq[26][64] = 567; + Big5Freq[7][19] = 566; + Big5Freq[4][12] = 565; + Big5Freq[11][124] = 564; + Big5Freq[7][89] = 563; + Big5Freq[15][124] = 562; + Big5Freq[4][108] = 561; + Big5Freq[19][66] = 560; + Big5Freq[3][21] = 559; + Big5Freq[24][12] = 558; + Big5Freq[28][111] = 557; + Big5Freq[12][107] = 556; + Big5Freq[3][112] = 555; + Big5Freq[8][113] = 554; + Big5Freq[5][40] = 553; + Big5Freq[26][145] = 552; + Big5Freq[3][48] = 551; + Big5Freq[3][70] = 550; + Big5Freq[22][17] = 549; + Big5Freq[16][47] = 548; + Big5Freq[3][53] = 547; + Big5Freq[4][24] = 546; + Big5Freq[32][120] = 545; + Big5Freq[24][49] = 544; + Big5Freq[24][142] = 543; + Big5Freq[18][66] = 542; + Big5Freq[29][150] = 541; + Big5Freq[5][122] = 540; + Big5Freq[5][114] = 539; + Big5Freq[3][44] = 538; + Big5Freq[10][128] = 537; + Big5Freq[15][20] = 536; + Big5Freq[13][33] = 535; + Big5Freq[14][87] = 534; + Big5Freq[3][126] = 533; + Big5Freq[4][53] = 532; + Big5Freq[4][40] = 531; + Big5Freq[9][93] = 530; + Big5Freq[15][137] = 529; + Big5Freq[10][123] = 528; + Big5Freq[4][56] = 527; + Big5Freq[5][71] = 526; + Big5Freq[10][8] = 525; + Big5Freq[5][16] = 524; + Big5Freq[5][146] = 523; + Big5Freq[18][88] = 522; + Big5Freq[24][4] = 521; + Big5Freq[20][47] = 520; + Big5Freq[5][33] = 519; + Big5Freq[9][43] = 518; + Big5Freq[20][12] = 517; + Big5Freq[20][13] = 516; + Big5Freq[5][156] = 515; + Big5Freq[22][140] = 514; + Big5Freq[8][146] = 513; + Big5Freq[21][123] = 512; + Big5Freq[4][90] = 511; + Big5Freq[5][62] = 510; + Big5Freq[17][59] = 509; + Big5Freq[10][37] = 508; + Big5Freq[18][107] = 507; + Big5Freq[14][53] = 506; + Big5Freq[22][51] = 505; + Big5Freq[8][13] = 504; + Big5Freq[5][29] = 503; + Big5Freq[9][7] = 502; + Big5Freq[22][14] = 501; + Big5Freq[8][55] = 500; + Big5Freq[33][9] = 499; + Big5Freq[16][64] = 498; + Big5Freq[7][131] = 497; + Big5Freq[34][4] = 496; + Big5Freq[7][101] = 495; + Big5Freq[11][139] = 494; + Big5Freq[3][135] = 493; + Big5Freq[7][102] = 492; + Big5Freq[17][13] = 491; + Big5Freq[3][20] = 490; + Big5Freq[27][106] = 489; + Big5Freq[5][88] = 488; + Big5Freq[6][33] = 487; + Big5Freq[5][139] = 486; + Big5Freq[6][0] = 485; + Big5Freq[17][58] = 484; + Big5Freq[5][133] = 483; + Big5Freq[9][107] = 482; + Big5Freq[23][39] = 481; + Big5Freq[5][23] = 480; + Big5Freq[3][79] = 479; + Big5Freq[32][97] = 478; + Big5Freq[3][136] = 477; + Big5Freq[4][94] = 476; + Big5Freq[21][61] = 475; + Big5Freq[23][123] = 474; + Big5Freq[26][16] = 473; + Big5Freq[24][137] = 472; + Big5Freq[22][18] = 471; + Big5Freq[5][1] = 470; + Big5Freq[20][119] = 469; + Big5Freq[3][7] = 468; + Big5Freq[10][79] = 467; + Big5Freq[15][105] = 466; + Big5Freq[3][144] = 465; + Big5Freq[12][80] = 464; + Big5Freq[15][73] = 463; + Big5Freq[3][19] = 462; + Big5Freq[8][109] = 461; + Big5Freq[3][15] = 460; + Big5Freq[31][82] = 459; + Big5Freq[3][43] = 458; + Big5Freq[25][119] = 457; + Big5Freq[16][111] = 456; + Big5Freq[7][77] = 455; + Big5Freq[3][95] = 454; + Big5Freq[24][82] = 453; + Big5Freq[7][52] = 452; + Big5Freq[9][151] = 451; + Big5Freq[3][129] = 450; + Big5Freq[5][87] = 449; + Big5Freq[3][55] = 448; + Big5Freq[8][153] = 447; + Big5Freq[4][83] = 446; + Big5Freq[3][114] = 445; + Big5Freq[23][147] = 444; + Big5Freq[15][31] = 443; + Big5Freq[3][54] = 442; + Big5Freq[11][122] = 441; + Big5Freq[4][4] = 440; + Big5Freq[34][149] = 439; + Big5Freq[3][17] = 438; + Big5Freq[21][64] = 437; + Big5Freq[26][144] = 436; + Big5Freq[4][62] = 435; + Big5Freq[8][15] = 434; + Big5Freq[35][80] = 433; + Big5Freq[7][110] = 432; + Big5Freq[23][114] = 431; + Big5Freq[3][108] = 430; + Big5Freq[3][62] = 429; + Big5Freq[21][41] = 428; + Big5Freq[15][99] = 427; + Big5Freq[5][47] = 426; + Big5Freq[4][96] = 425; + Big5Freq[20][122] = 424; + Big5Freq[5][21] = 423; + Big5Freq[4][157] = 422; + Big5Freq[16][14] = 421; + Big5Freq[3][117] = 420; + Big5Freq[7][129] = 419; + Big5Freq[4][27] = 418; + Big5Freq[5][30] = 417; + Big5Freq[22][16] = 416; + Big5Freq[5][64] = 415; + Big5Freq[17][99] = 414; + Big5Freq[17][57] = 413; + Big5Freq[8][105] = 412; + Big5Freq[5][112] = 411; + Big5Freq[20][59] = 410; + Big5Freq[6][129] = 409; + Big5Freq[18][17] = 408; + Big5Freq[3][92] = 407; + Big5Freq[28][118] = 406; + Big5Freq[3][109] = 405; + Big5Freq[31][51] = 404; + Big5Freq[13][116] = 403; + Big5Freq[6][15] = 402; + Big5Freq[36][136] = 401; + Big5Freq[12][74] = 400; + Big5Freq[20][88] = 399; + Big5Freq[36][68] = 398; + Big5Freq[3][147] = 397; + Big5Freq[15][84] = 396; + Big5Freq[16][32] = 395; + Big5Freq[16][58] = 394; + Big5Freq[7][66] = 393; + Big5Freq[23][107] = 392; + Big5Freq[9][6] = 391; + Big5Freq[12][86] = 390; + Big5Freq[23][112] = 389; + Big5Freq[37][23] = 388; + Big5Freq[3][138] = 387; + Big5Freq[20][68] = 386; + Big5Freq[15][116] = 385; + Big5Freq[18][64] = 384; + Big5Freq[12][139] = 383; + Big5Freq[11][155] = 382; + Big5Freq[4][156] = 381; + Big5Freq[12][84] = 380; + Big5Freq[18][49] = 379; + Big5Freq[25][125] = 378; + Big5Freq[25][147] = 377; + Big5Freq[15][110] = 376; + Big5Freq[19][96] = 375; + Big5Freq[30][152] = 374; + Big5Freq[6][31] = 373; + Big5Freq[27][117] = 372; + Big5Freq[3][10] = 371; + Big5Freq[6][131] = 370; + Big5Freq[13][112] = 369; + Big5Freq[36][156] = 368; + Big5Freq[4][60] = 367; + Big5Freq[15][121] = 366; + Big5Freq[4][112] = 365; + Big5Freq[30][142] = 364; + Big5Freq[23][154] = 363; + Big5Freq[27][101] = 362; + Big5Freq[9][140] = 361; + Big5Freq[3][89] = 360; + Big5Freq[18][148] = 359; + Big5Freq[4][69] = 358; + Big5Freq[16][49] = 357; + Big5Freq[6][117] = 356; + Big5Freq[36][55] = 355; + Big5Freq[5][123] = 354; + Big5Freq[4][126] = 353; + Big5Freq[4][119] = 352; + Big5Freq[9][95] = 351; + Big5Freq[5][24] = 350; + Big5Freq[16][133] = 349; + Big5Freq[10][134] = 348; + Big5Freq[26][59] = 347; + Big5Freq[6][41] = 346; + Big5Freq[6][146] = 345; + Big5Freq[19][24] = 344; + Big5Freq[5][113] = 343; + Big5Freq[10][118] = 342; + Big5Freq[34][151] = 341; + Big5Freq[9][72] = 340; + Big5Freq[31][25] = 339; + Big5Freq[18][126] = 338; + Big5Freq[18][28] = 337; + Big5Freq[4][153] = 336; + Big5Freq[3][84] = 335; + Big5Freq[21][18] = 334; + Big5Freq[25][129] = 333; + Big5Freq[6][107] = 332; + Big5Freq[12][25] = 331; + Big5Freq[17][109] = 330; + Big5Freq[7][76] = 329; + Big5Freq[15][15] = 328; + Big5Freq[4][14] = 327; + Big5Freq[23][88] = 326; + Big5Freq[18][2] = 325; + Big5Freq[6][88] = 324; + Big5Freq[16][84] = 323; + Big5Freq[12][48] = 322; + Big5Freq[7][68] = 321; + Big5Freq[5][50] = 320; + Big5Freq[13][54] = 319; + Big5Freq[7][98] = 318; + Big5Freq[11][6] = 317; + Big5Freq[9][80] = 316; + Big5Freq[16][41] = 315; + Big5Freq[7][43] = 314; + Big5Freq[28][117] = 313; + Big5Freq[3][51] = 312; + Big5Freq[7][3] = 311; + Big5Freq[20][81] = 310; + Big5Freq[4][2] = 309; + Big5Freq[11][16] = 308; + Big5Freq[10][4] = 307; + Big5Freq[10][119] = 306; + Big5Freq[6][142] = 305; + Big5Freq[18][51] = 304; + Big5Freq[8][144] = 303; + Big5Freq[10][65] = 302; + Big5Freq[11][64] = 301; + Big5Freq[11][130] = 300; + Big5Freq[9][92] = 299; + Big5Freq[18][29] = 298; + Big5Freq[18][78] = 297; + Big5Freq[18][151] = 296; + Big5Freq[33][127] = 295; + Big5Freq[35][113] = 294; + Big5Freq[10][155] = 293; + Big5Freq[3][76] = 292; + Big5Freq[36][123] = 291; + Big5Freq[13][143] = 290; + Big5Freq[5][135] = 289; + Big5Freq[23][116] = 288; + Big5Freq[6][101] = 287; + Big5Freq[14][74] = 286; + Big5Freq[7][153] = 285; + Big5Freq[3][101] = 284; + Big5Freq[9][74] = 283; + Big5Freq[3][156] = 282; + Big5Freq[4][147] = 281; + Big5Freq[9][12] = 280; + Big5Freq[18][133] = 279; + Big5Freq[4][0] = 278; + Big5Freq[7][155] = 277; + Big5Freq[9][144] = 276; + Big5Freq[23][49] = 275; + Big5Freq[5][89] = 274; + Big5Freq[10][11] = 273; + Big5Freq[3][110] = 272; + Big5Freq[3][40] = 271; + Big5Freq[29][115] = 270; + Big5Freq[9][100] = 269; + Big5Freq[21][67] = 268; + Big5Freq[23][145] = 267; + Big5Freq[10][47] = 266; + Big5Freq[4][31] = 265; + Big5Freq[4][81] = 264; + Big5Freq[22][62] = 263; + Big5Freq[4][28] = 262; + Big5Freq[27][39] = 261; + Big5Freq[27][54] = 260; + Big5Freq[32][46] = 259; + Big5Freq[4][76] = 258; + Big5Freq[26][15] = 257; + Big5Freq[12][154] = 256; + Big5Freq[9][150] = 255; + Big5Freq[15][17] = 254; + Big5Freq[5][129] = 253; + Big5Freq[10][40] = 252; + Big5Freq[13][37] = 251; + Big5Freq[31][104] = 250; + Big5Freq[3][152] = 249; + Big5Freq[5][22] = 248; + Big5Freq[8][48] = 247; + Big5Freq[4][74] = 246; + Big5Freq[6][17] = 245; + Big5Freq[30][82] = 244; + Big5Freq[4][116] = 243; + Big5Freq[16][42] = 242; + Big5Freq[5][55] = 241; + Big5Freq[4][64] = 240; + Big5Freq[14][19] = 239; + Big5Freq[35][82] = 238; + Big5Freq[30][139] = 237; + Big5Freq[26][152] = 236; + Big5Freq[32][32] = 235; + Big5Freq[21][102] = 234; + Big5Freq[10][131] = 233; + Big5Freq[9][128] = 232; + Big5Freq[3][87] = 231; + Big5Freq[4][51] = 230; + Big5Freq[10][15] = 229; + Big5Freq[4][150] = 228; + Big5Freq[7][4] = 227; + Big5Freq[7][51] = 226; + Big5Freq[7][157] = 225; + Big5Freq[4][146] = 224; + Big5Freq[4][91] = 223; + Big5Freq[7][13] = 222; + Big5Freq[17][116] = 221; + Big5Freq[23][21] = 220; + Big5Freq[5][106] = 219; + Big5Freq[14][100] = 218; + Big5Freq[10][152] = 217; + Big5Freq[14][89] = 216; + Big5Freq[6][138] = 215; + Big5Freq[12][157] = 214; + Big5Freq[10][102] = 213; + Big5Freq[19][94] = 212; + Big5Freq[7][74] = 211; + Big5Freq[18][128] = 210; + Big5Freq[27][111] = 209; + Big5Freq[11][57] = 208; + Big5Freq[3][131] = 207; + Big5Freq[30][23] = 206; + Big5Freq[30][126] = 205; + Big5Freq[4][36] = 204; + Big5Freq[26][124] = 203; + Big5Freq[4][19] = 202; + Big5Freq[9][152] = 201; + /* + * Big5Freq[5][0] = 200; Big5Freq[26][57] = 199; Big5Freq[13][155] = 198; Big5Freq[3][38] = 197; Big5Freq[9][155] = 196; + * Big5Freq[28][53] = 195; Big5Freq[15][71] = 194; Big5Freq[21][95] = 193; Big5Freq[15][112] = 192; Big5Freq[14][138] = 191; + * Big5Freq[8][18] = 190; Big5Freq[20][151] = 189; Big5Freq[37][27] = 188; Big5Freq[32][48] = 187; Big5Freq[23][66] = 186; + * Big5Freq[9][2] = 185; Big5Freq[13][133] = 184; Big5Freq[7][127] = 183; Big5Freq[3][11] = 182; Big5Freq[12][118] = 181; + * Big5Freq[13][101] = 180; Big5Freq[30][153] = 179; Big5Freq[4][65] = 178; Big5Freq[5][25] = 177; Big5Freq[5][140] = 176; + * Big5Freq[6][25] = 175; Big5Freq[4][52] = 174; Big5Freq[30][156] = 173; Big5Freq[16][13] = 172; Big5Freq[21][8] = 171; + * Big5Freq[19][74] = 170; Big5Freq[15][145] = 169; Big5Freq[9][15] = 168; Big5Freq[13][82] = 167; Big5Freq[26][86] = 166; + * Big5Freq[18][52] = 165; Big5Freq[6][109] = 164; Big5Freq[10][99] = 163; Big5Freq[18][101] = 162; Big5Freq[25][49] = 161; + * Big5Freq[31][79] = 160; Big5Freq[28][20] = 159; Big5Freq[12][115] = 158; Big5Freq[15][66] = 157; Big5Freq[11][104] = 156; + * Big5Freq[23][106] = 155; Big5Freq[34][157] = 154; Big5Freq[32][94] = 153; Big5Freq[29][88] = 152; Big5Freq[10][46] = 151; + * Big5Freq[13][118] = 150; Big5Freq[20][37] = 149; Big5Freq[12][30] = 148; Big5Freq[21][4] = 147; Big5Freq[16][33] = 146; + * Big5Freq[13][52] = 145; Big5Freq[4][7] = 144; Big5Freq[21][49] = 143; Big5Freq[3][27] = 142; Big5Freq[16][91] = 141; + * Big5Freq[5][155] = 140; Big5Freq[29][130] = 139; Big5Freq[3][125] = 138; Big5Freq[14][26] = 137; Big5Freq[15][39] = 136; + * Big5Freq[24][110] = 135; Big5Freq[7][141] = 134; Big5Freq[21][15] = 133; Big5Freq[32][104] = 132; Big5Freq[8][31] = 131; + * Big5Freq[34][112] = 130; Big5Freq[10][75] = 129; Big5Freq[21][23] = 128; Big5Freq[34][131] = 127; Big5Freq[12][3] = 126; + * Big5Freq[10][62] = 125; Big5Freq[9][120] = 124; Big5Freq[32][149] = 123; Big5Freq[8][44] = 122; Big5Freq[24][2] = 121; + * Big5Freq[6][148] = 120; Big5Freq[15][103] = 119; Big5Freq[36][54] = 118; Big5Freq[36][134] = 117; Big5Freq[11][7] = 116; + * Big5Freq[3][90] = 115; Big5Freq[36][73] = 114; Big5Freq[8][102] = 113; Big5Freq[12][87] = 112; Big5Freq[25][64] = 111; + * Big5Freq[9][1] = 110; Big5Freq[24][121] = 109; Big5Freq[5][75] = 108; Big5Freq[17][83] = 107; Big5Freq[18][57] = 106; + * Big5Freq[8][95] = 105; Big5Freq[14][36] = 104; Big5Freq[28][113] = 103; Big5Freq[12][56] = 102; Big5Freq[14][61] = 101; + * Big5Freq[25][138] = 100; Big5Freq[4][34] = 99; Big5Freq[11][152] = 98; Big5Freq[35][0] = 97; Big5Freq[4][15] = 96; + * Big5Freq[8][82] = 95; Big5Freq[20][73] = 94; Big5Freq[25][52] = 93; Big5Freq[24][6] = 92; Big5Freq[21][78] = 91; + * Big5Freq[17][32] = 90; Big5Freq[17][91] = 89; Big5Freq[5][76] = 88; Big5Freq[15][60] = 87; Big5Freq[15][150] = 86; + * Big5Freq[5][80] = 85; Big5Freq[15][81] = 84; Big5Freq[28][108] = 83; Big5Freq[18][14] = 82; Big5Freq[19][109] = 81; + * Big5Freq[28][133] = 80; Big5Freq[21][97] = 79; Big5Freq[5][105] = 78; Big5Freq[18][114] = 77; Big5Freq[16][95] = 76; + * Big5Freq[5][51] = 75; Big5Freq[3][148] = 74; Big5Freq[22][102] = 73; Big5Freq[4][123] = 72; Big5Freq[8][88] = 71; + * Big5Freq[25][111] = 70; Big5Freq[8][149] = 69; Big5Freq[9][48] = 68; Big5Freq[16][126] = 67; Big5Freq[33][150] = 66; + * Big5Freq[9][54] = 65; Big5Freq[29][104] = 64; Big5Freq[3][3] = 63; Big5Freq[11][49] = 62; Big5Freq[24][109] = 61; + * Big5Freq[28][116] = 60; Big5Freq[34][113] = 59; Big5Freq[5][3] = 58; Big5Freq[21][106] = 57; Big5Freq[4][98] = 56; + * Big5Freq[12][135] = 55; Big5Freq[16][101] = 54; Big5Freq[12][147] = 53; Big5Freq[27][55] = 52; Big5Freq[3][5] = 51; + * Big5Freq[11][101] = 50; Big5Freq[16][157] = 49; Big5Freq[22][114] = 48; Big5Freq[18][46] = 47; Big5Freq[4][29] = 46; + * Big5Freq[8][103] = 45; Big5Freq[16][151] = 44; Big5Freq[8][29] = 43; Big5Freq[15][114] = 42; Big5Freq[22][70] = 41; + * Big5Freq[13][121] = 40; Big5Freq[7][112] = 39; Big5Freq[20][83] = 38; Big5Freq[3][36] = 37; Big5Freq[10][103] = 36; + * Big5Freq[3][96] = 35; Big5Freq[21][79] = 34; Big5Freq[25][120] = 33; Big5Freq[29][121] = 32; Big5Freq[23][71] = 31; + * Big5Freq[21][22] = 30; Big5Freq[18][89] = 29; Big5Freq[25][104] = 28; Big5Freq[10][124] = 27; Big5Freq[26][4] = 26; + * Big5Freq[21][136] = 25; Big5Freq[6][112] = 24; Big5Freq[12][103] = 23; Big5Freq[17][66] = 22; Big5Freq[13][151] = 21; + * Big5Freq[33][152] = 20; Big5Freq[11][148] = 19; Big5Freq[13][57] = 18; Big5Freq[13][41] = 17; Big5Freq[7][60] = 16; + * Big5Freq[21][29] = 15; Big5Freq[9][157] = 14; Big5Freq[24][95] = 13; Big5Freq[15][148] = 12; Big5Freq[15][122] = 11; + * Big5Freq[6][125] = 10; Big5Freq[11][25] = 9; Big5Freq[20][55] = 8; Big5Freq[19][84] = 7; Big5Freq[21][82] = 6; + * Big5Freq[24][3] = 5; Big5Freq[13][70] = 4; Big5Freq[6][21] = 3; Big5Freq[21][86] = 2; Big5Freq[12][23] = 1; + * Big5Freq[3][85] = 0; EUC_TWFreq[45][90] = 600; + */ + Big5PFreq[41][122] = 600; + Big5PFreq[35][0] = 599; + Big5PFreq[43][15] = 598; + Big5PFreq[35][99] = 597; + Big5PFreq[35][6] = 596; + Big5PFreq[35][8] = 595; + Big5PFreq[38][154] = 594; + Big5PFreq[37][34] = 593; + Big5PFreq[37][115] = 592; + Big5PFreq[36][12] = 591; + Big5PFreq[18][77] = 590; + Big5PFreq[35][100] = 589; + Big5PFreq[35][42] = 588; + Big5PFreq[120][75] = 587; + Big5PFreq[35][23] = 586; + Big5PFreq[13][72] = 585; + Big5PFreq[0][67] = 584; + Big5PFreq[39][172] = 583; + Big5PFreq[22][182] = 582; + Big5PFreq[15][186] = 581; + Big5PFreq[15][165] = 580; + Big5PFreq[35][44] = 579; + Big5PFreq[40][13] = 578; + Big5PFreq[38][1] = 577; + Big5PFreq[37][33] = 576; + Big5PFreq[36][24] = 575; + Big5PFreq[56][4] = 574; + Big5PFreq[35][29] = 573; + Big5PFreq[9][96] = 572; + Big5PFreq[37][62] = 571; + Big5PFreq[48][47] = 570; + Big5PFreq[51][14] = 569; + Big5PFreq[39][122] = 568; + Big5PFreq[44][46] = 567; + Big5PFreq[35][21] = 566; + Big5PFreq[36][8] = 565; + Big5PFreq[36][141] = 564; + Big5PFreq[3][81] = 563; + Big5PFreq[37][155] = 562; + Big5PFreq[42][84] = 561; + Big5PFreq[36][40] = 560; + Big5PFreq[35][103] = 559; + Big5PFreq[11][84] = 558; + Big5PFreq[45][33] = 557; + Big5PFreq[121][79] = 556; + Big5PFreq[2][77] = 555; + Big5PFreq[36][41] = 554; + Big5PFreq[37][47] = 553; + Big5PFreq[39][125] = 552; + Big5PFreq[37][26] = 551; + Big5PFreq[35][48] = 550; + Big5PFreq[35][28] = 549; + Big5PFreq[35][159] = 548; + Big5PFreq[37][40] = 547; + Big5PFreq[35][145] = 546; + Big5PFreq[37][147] = 545; + Big5PFreq[46][160] = 544; + Big5PFreq[37][46] = 543; + Big5PFreq[50][99] = 542; + Big5PFreq[52][13] = 541; + Big5PFreq[10][82] = 540; + Big5PFreq[35][169] = 539; + Big5PFreq[35][31] = 538; + Big5PFreq[47][31] = 537; + Big5PFreq[18][79] = 536; + Big5PFreq[16][113] = 535; + Big5PFreq[37][104] = 534; + Big5PFreq[39][134] = 533; + Big5PFreq[36][53] = 532; + Big5PFreq[38][0] = 531; + Big5PFreq[4][86] = 530; + Big5PFreq[54][17] = 529; + Big5PFreq[43][157] = 528; + Big5PFreq[35][165] = 527; + Big5PFreq[69][147] = 526; + Big5PFreq[117][95] = 525; + Big5PFreq[35][162] = 524; + Big5PFreq[35][17] = 523; + Big5PFreq[36][142] = 522; + Big5PFreq[36][4] = 521; + Big5PFreq[37][166] = 520; + Big5PFreq[35][168] = 519; + Big5PFreq[35][19] = 518; + Big5PFreq[37][48] = 517; + Big5PFreq[42][37] = 516; + Big5PFreq[40][146] = 515; + Big5PFreq[36][123] = 514; + Big5PFreq[22][41] = 513; + Big5PFreq[20][119] = 512; + Big5PFreq[2][74] = 511; + Big5PFreq[44][113] = 510; + Big5PFreq[35][125] = 509; + Big5PFreq[37][16] = 508; + Big5PFreq[35][20] = 507; + Big5PFreq[35][55] = 506; + Big5PFreq[37][145] = 505; + Big5PFreq[0][88] = 504; + Big5PFreq[3][94] = 503; + Big5PFreq[6][65] = 502; + Big5PFreq[26][15] = 501; + Big5PFreq[41][126] = 500; + Big5PFreq[36][129] = 499; + Big5PFreq[31][75] = 498; + Big5PFreq[19][61] = 497; + Big5PFreq[35][128] = 496; + Big5PFreq[29][79] = 495; + Big5PFreq[36][62] = 494; + Big5PFreq[37][189] = 493; + Big5PFreq[39][109] = 492; + Big5PFreq[39][135] = 491; + Big5PFreq[72][15] = 490; + Big5PFreq[47][106] = 489; + Big5PFreq[54][14] = 488; + Big5PFreq[24][52] = 487; + Big5PFreq[38][162] = 486; + Big5PFreq[41][43] = 485; + Big5PFreq[37][121] = 484; + Big5PFreq[14][66] = 483; + Big5PFreq[37][30] = 482; + Big5PFreq[35][7] = 481; + Big5PFreq[49][58] = 480; + Big5PFreq[43][188] = 479; + Big5PFreq[24][66] = 478; + Big5PFreq[35][171] = 477; + Big5PFreq[40][186] = 476; + Big5PFreq[39][164] = 475; + Big5PFreq[78][186] = 474; + Big5PFreq[8][72] = 473; + Big5PFreq[36][190] = 472; + Big5PFreq[35][53] = 471; + Big5PFreq[35][54] = 470; + Big5PFreq[22][159] = 469; + Big5PFreq[35][9] = 468; + Big5PFreq[41][140] = 467; + Big5PFreq[37][22] = 466; + Big5PFreq[48][97] = 465; + Big5PFreq[50][97] = 464; + Big5PFreq[36][127] = 463; + Big5PFreq[37][23] = 462; + Big5PFreq[40][55] = 461; + Big5PFreq[35][43] = 460; + Big5PFreq[26][22] = 459; + Big5PFreq[35][15] = 458; + Big5PFreq[72][179] = 457; + Big5PFreq[20][129] = 456; + Big5PFreq[52][101] = 455; + Big5PFreq[35][12] = 454; + Big5PFreq[42][156] = 453; + Big5PFreq[15][157] = 452; + Big5PFreq[50][140] = 451; + Big5PFreq[26][28] = 450; + Big5PFreq[54][51] = 449; + Big5PFreq[35][112] = 448; + Big5PFreq[36][116] = 447; + Big5PFreq[42][11] = 446; + Big5PFreq[37][172] = 445; + Big5PFreq[37][29] = 444; + Big5PFreq[44][107] = 443; + Big5PFreq[50][17] = 442; + Big5PFreq[39][107] = 441; + Big5PFreq[19][109] = 440; + Big5PFreq[36][60] = 439; + Big5PFreq[49][132] = 438; + Big5PFreq[26][16] = 437; + Big5PFreq[43][155] = 436; + Big5PFreq[37][120] = 435; + Big5PFreq[15][159] = 434; + Big5PFreq[43][6] = 433; + Big5PFreq[45][188] = 432; + Big5PFreq[35][38] = 431; + Big5PFreq[39][143] = 430; + Big5PFreq[48][144] = 429; + Big5PFreq[37][168] = 428; + Big5PFreq[37][1] = 427; + Big5PFreq[36][109] = 426; + Big5PFreq[46][53] = 425; + Big5PFreq[38][54] = 424; + Big5PFreq[36][0] = 423; + Big5PFreq[72][33] = 422; + Big5PFreq[42][8] = 421; + Big5PFreq[36][31] = 420; + Big5PFreq[35][150] = 419; + Big5PFreq[118][93] = 418; + Big5PFreq[37][61] = 417; + Big5PFreq[0][85] = 416; + Big5PFreq[36][27] = 415; + Big5PFreq[35][134] = 414; + Big5PFreq[36][145] = 413; + Big5PFreq[6][96] = 412; + Big5PFreq[36][14] = 411; + Big5PFreq[16][36] = 410; + Big5PFreq[15][175] = 409; + Big5PFreq[35][10] = 408; + Big5PFreq[36][189] = 407; + Big5PFreq[35][51] = 406; + Big5PFreq[35][109] = 405; + Big5PFreq[35][147] = 404; + Big5PFreq[35][180] = 403; + Big5PFreq[72][5] = 402; + Big5PFreq[36][107] = 401; + Big5PFreq[49][116] = 400; + Big5PFreq[73][30] = 399; + Big5PFreq[6][90] = 398; + Big5PFreq[2][70] = 397; + Big5PFreq[17][141] = 396; + Big5PFreq[35][62] = 395; + Big5PFreq[16][180] = 394; + Big5PFreq[4][91] = 393; + Big5PFreq[15][171] = 392; + Big5PFreq[35][177] = 391; + Big5PFreq[37][173] = 390; + Big5PFreq[16][121] = 389; + Big5PFreq[35][5] = 388; + Big5PFreq[46][122] = 387; + Big5PFreq[40][138] = 386; + Big5PFreq[50][49] = 385; + Big5PFreq[36][152] = 384; + Big5PFreq[13][43] = 383; + Big5PFreq[9][88] = 382; + Big5PFreq[36][159] = 381; + Big5PFreq[27][62] = 380; + Big5PFreq[40][18] = 379; + Big5PFreq[17][129] = 378; + Big5PFreq[43][97] = 377; + Big5PFreq[13][131] = 376; + Big5PFreq[46][107] = 375; + Big5PFreq[60][64] = 374; + Big5PFreq[36][179] = 373; + Big5PFreq[37][55] = 372; + Big5PFreq[41][173] = 371; + Big5PFreq[44][172] = 370; + Big5PFreq[23][187] = 369; + Big5PFreq[36][149] = 368; + Big5PFreq[17][125] = 367; + Big5PFreq[55][180] = 366; + Big5PFreq[51][129] = 365; + Big5PFreq[36][51] = 364; + Big5PFreq[37][122] = 363; + Big5PFreq[48][32] = 362; + Big5PFreq[51][99] = 361; + Big5PFreq[54][16] = 360; + Big5PFreq[41][183] = 359; + Big5PFreq[37][179] = 358; + Big5PFreq[38][179] = 357; + Big5PFreq[35][143] = 356; + Big5PFreq[37][24] = 355; + Big5PFreq[40][177] = 354; + Big5PFreq[47][117] = 353; + Big5PFreq[39][52] = 352; + Big5PFreq[22][99] = 351; + Big5PFreq[40][142] = 350; + Big5PFreq[36][49] = 349; + Big5PFreq[38][17] = 348; + Big5PFreq[39][188] = 347; + Big5PFreq[36][186] = 346; + Big5PFreq[35][189] = 345; + Big5PFreq[41][7] = 344; + Big5PFreq[18][91] = 343; + Big5PFreq[43][137] = 342; + Big5PFreq[35][142] = 341; + Big5PFreq[35][117] = 340; + Big5PFreq[39][138] = 339; + Big5PFreq[16][59] = 338; + Big5PFreq[39][174] = 337; + Big5PFreq[55][145] = 336; + Big5PFreq[37][21] = 335; + Big5PFreq[36][180] = 334; + Big5PFreq[37][156] = 333; + Big5PFreq[49][13] = 332; + Big5PFreq[41][107] = 331; + Big5PFreq[36][56] = 330; + Big5PFreq[53][8] = 329; + Big5PFreq[22][114] = 328; + Big5PFreq[5][95] = 327; + Big5PFreq[37][0] = 326; + Big5PFreq[26][183] = 325; + Big5PFreq[22][66] = 324; + Big5PFreq[35][58] = 323; + Big5PFreq[48][117] = 322; + Big5PFreq[36][102] = 321; + Big5PFreq[22][122] = 320; + Big5PFreq[35][11] = 319; + Big5PFreq[46][19] = 318; + Big5PFreq[22][49] = 317; + Big5PFreq[48][166] = 316; + Big5PFreq[41][125] = 315; + Big5PFreq[41][1] = 314; + Big5PFreq[35][178] = 313; + Big5PFreq[41][12] = 312; + Big5PFreq[26][167] = 311; + Big5PFreq[42][152] = 310; + Big5PFreq[42][46] = 309; + Big5PFreq[42][151] = 308; + Big5PFreq[20][135] = 307; + Big5PFreq[37][162] = 306; + Big5PFreq[37][50] = 305; + Big5PFreq[22][185] = 304; + Big5PFreq[36][166] = 303; + Big5PFreq[19][40] = 302; + Big5PFreq[22][107] = 301; + Big5PFreq[22][102] = 300; + Big5PFreq[57][162] = 299; + Big5PFreq[22][124] = 298; + Big5PFreq[37][138] = 297; + Big5PFreq[37][25] = 296; + Big5PFreq[0][69] = 295; + Big5PFreq[43][172] = 294; + Big5PFreq[42][167] = 293; + Big5PFreq[35][120] = 292; + Big5PFreq[41][128] = 291; + Big5PFreq[2][88] = 290; + Big5PFreq[20][123] = 289; + Big5PFreq[35][123] = 288; + Big5PFreq[36][28] = 287; + Big5PFreq[42][188] = 286; + Big5PFreq[42][164] = 285; + Big5PFreq[42][4] = 284; + Big5PFreq[43][57] = 283; + Big5PFreq[39][3] = 282; + Big5PFreq[42][3] = 281; + Big5PFreq[57][158] = 280; + Big5PFreq[35][146] = 279; + Big5PFreq[24][54] = 278; + Big5PFreq[13][110] = 277; + Big5PFreq[23][132] = 276; + Big5PFreq[26][102] = 275; + Big5PFreq[55][178] = 274; + Big5PFreq[17][117] = 273; + Big5PFreq[41][161] = 272; + Big5PFreq[38][150] = 271; + Big5PFreq[10][71] = 270; + Big5PFreq[47][60] = 269; + Big5PFreq[16][114] = 268; + Big5PFreq[21][47] = 267; + Big5PFreq[39][101] = 266; + Big5PFreq[18][45] = 265; + Big5PFreq[40][121] = 264; + Big5PFreq[45][41] = 263; + Big5PFreq[22][167] = 262; + Big5PFreq[26][149] = 261; + Big5PFreq[15][189] = 260; + Big5PFreq[41][177] = 259; + Big5PFreq[46][36] = 258; + Big5PFreq[20][40] = 257; + Big5PFreq[41][54] = 256; + Big5PFreq[3][87] = 255; + Big5PFreq[40][16] = 254; + Big5PFreq[42][15] = 253; + Big5PFreq[11][83] = 252; + Big5PFreq[0][94] = 251; + Big5PFreq[122][81] = 250; + Big5PFreq[41][26] = 249; + Big5PFreq[36][34] = 248; + Big5PFreq[44][148] = 247; + Big5PFreq[35][3] = 246; + Big5PFreq[36][114] = 245; + Big5PFreq[42][112] = 244; + Big5PFreq[35][183] = 243; + Big5PFreq[49][73] = 242; + Big5PFreq[39][2] = 241; + Big5PFreq[38][121] = 240; + Big5PFreq[44][114] = 239; + Big5PFreq[49][32] = 238; + Big5PFreq[1][65] = 237; + Big5PFreq[38][25] = 236; + Big5PFreq[39][4] = 235; + Big5PFreq[42][62] = 234; + Big5PFreq[35][40] = 233; + Big5PFreq[24][2] = 232; + Big5PFreq[53][49] = 231; + Big5PFreq[41][133] = 230; + Big5PFreq[43][134] = 229; + Big5PFreq[3][83] = 228; + Big5PFreq[38][158] = 227; + Big5PFreq[24][17] = 226; + Big5PFreq[52][59] = 225; + Big5PFreq[38][41] = 224; + Big5PFreq[37][127] = 223; + Big5PFreq[22][175] = 222; + Big5PFreq[44][30] = 221; + Big5PFreq[47][178] = 220; + Big5PFreq[43][99] = 219; + Big5PFreq[19][4] = 218; + Big5PFreq[37][97] = 217; + Big5PFreq[38][181] = 216; + Big5PFreq[45][103] = 215; + Big5PFreq[1][86] = 214; + Big5PFreq[40][15] = 213; + Big5PFreq[22][136] = 212; + Big5PFreq[75][165] = 211; + Big5PFreq[36][15] = 210; + Big5PFreq[46][80] = 209; + Big5PFreq[59][55] = 208; + Big5PFreq[37][108] = 207; + Big5PFreq[21][109] = 206; + Big5PFreq[24][165] = 205; + Big5PFreq[79][158] = 204; + Big5PFreq[44][139] = 203; + Big5PFreq[36][124] = 202; + Big5PFreq[42][185] = 201; + Big5PFreq[39][186] = 200; + Big5PFreq[22][128] = 199; + Big5PFreq[40][44] = 198; + Big5PFreq[41][105] = 197; + Big5PFreq[1][70] = 196; + Big5PFreq[1][68] = 195; + Big5PFreq[53][22] = 194; + Big5PFreq[36][54] = 193; + Big5PFreq[47][147] = 192; + Big5PFreq[35][36] = 191; + Big5PFreq[35][185] = 190; + Big5PFreq[45][37] = 189; + Big5PFreq[43][163] = 188; + Big5PFreq[56][115] = 187; + Big5PFreq[38][164] = 186; + Big5PFreq[35][141] = 185; + Big5PFreq[42][132] = 184; + Big5PFreq[46][120] = 183; + Big5PFreq[69][142] = 182; + Big5PFreq[38][175] = 181; + Big5PFreq[22][112] = 180; + Big5PFreq[38][142] = 179; + Big5PFreq[40][37] = 178; + Big5PFreq[37][109] = 177; + Big5PFreq[40][144] = 176; + Big5PFreq[44][117] = 175; + Big5PFreq[35][181] = 174; + Big5PFreq[26][105] = 173; + Big5PFreq[16][48] = 172; + Big5PFreq[44][122] = 171; + Big5PFreq[12][86] = 170; + Big5PFreq[84][53] = 169; + Big5PFreq[17][44] = 168; + Big5PFreq[59][54] = 167; + Big5PFreq[36][98] = 166; + Big5PFreq[45][115] = 165; + Big5PFreq[73][9] = 164; + Big5PFreq[44][123] = 163; + Big5PFreq[37][188] = 162; + Big5PFreq[51][117] = 161; + Big5PFreq[15][156] = 160; + Big5PFreq[36][155] = 159; + Big5PFreq[44][25] = 158; + Big5PFreq[38][12] = 157; + Big5PFreq[38][140] = 156; + Big5PFreq[23][4] = 155; + Big5PFreq[45][149] = 154; + Big5PFreq[22][189] = 153; + Big5PFreq[38][147] = 152; + Big5PFreq[27][5] = 151; + Big5PFreq[22][42] = 150; + Big5PFreq[3][68] = 149; + Big5PFreq[39][51] = 148; + Big5PFreq[36][29] = 147; + Big5PFreq[20][108] = 146; + Big5PFreq[50][57] = 145; + Big5PFreq[55][104] = 144; + Big5PFreq[22][46] = 143; + Big5PFreq[18][164] = 142; + Big5PFreq[50][159] = 141; + Big5PFreq[85][131] = 140; + Big5PFreq[26][79] = 139; + Big5PFreq[38][100] = 138; + Big5PFreq[53][112] = 137; + Big5PFreq[20][190] = 136; + Big5PFreq[14][69] = 135; + Big5PFreq[23][11] = 134; + Big5PFreq[40][114] = 133; + Big5PFreq[40][148] = 132; + Big5PFreq[53][130] = 131; + Big5PFreq[36][2] = 130; + Big5PFreq[66][82] = 129; + Big5PFreq[45][166] = 128; + Big5PFreq[4][88] = 127; + Big5PFreq[16][57] = 126; + Big5PFreq[22][116] = 125; + Big5PFreq[36][108] = 124; + Big5PFreq[13][48] = 123; + Big5PFreq[54][12] = 122; + Big5PFreq[40][136] = 121; + Big5PFreq[36][128] = 120; + Big5PFreq[23][6] = 119; + Big5PFreq[38][125] = 118; + Big5PFreq[45][154] = 117; + Big5PFreq[51][127] = 116; + Big5PFreq[44][163] = 115; + Big5PFreq[16][173] = 114; + Big5PFreq[43][49] = 113; + Big5PFreq[20][112] = 112; + Big5PFreq[15][168] = 111; + Big5PFreq[35][129] = 110; + Big5PFreq[20][45] = 109; + Big5PFreq[38][10] = 108; + Big5PFreq[57][171] = 107; + Big5PFreq[44][190] = 106; + Big5PFreq[40][56] = 105; + Big5PFreq[36][156] = 104; + Big5PFreq[3][88] = 103; + Big5PFreq[50][122] = 102; + Big5PFreq[36][7] = 101; + Big5PFreq[39][43] = 100; + Big5PFreq[15][166] = 99; + Big5PFreq[42][136] = 98; + Big5PFreq[22][131] = 97; + Big5PFreq[44][23] = 96; + Big5PFreq[54][147] = 95; + Big5PFreq[41][32] = 94; + Big5PFreq[23][121] = 93; + Big5PFreq[39][108] = 92; + Big5PFreq[2][78] = 91; + Big5PFreq[40][155] = 90; + Big5PFreq[55][51] = 89; + Big5PFreq[19][34] = 88; + Big5PFreq[48][128] = 87; + Big5PFreq[48][159] = 86; + Big5PFreq[20][70] = 85; + Big5PFreq[34][71] = 84; + Big5PFreq[16][31] = 83; + Big5PFreq[42][157] = 82; + Big5PFreq[20][44] = 81; + Big5PFreq[11][92] = 80; + Big5PFreq[44][180] = 79; + Big5PFreq[84][33] = 78; + Big5PFreq[16][116] = 77; + Big5PFreq[61][163] = 76; + Big5PFreq[35][164] = 75; + Big5PFreq[36][42] = 74; + Big5PFreq[13][40] = 73; + Big5PFreq[43][176] = 72; + Big5PFreq[2][66] = 71; + Big5PFreq[20][133] = 70; + Big5PFreq[36][65] = 69; + Big5PFreq[38][33] = 68; + Big5PFreq[12][91] = 67; + Big5PFreq[36][26] = 66; + Big5PFreq[15][174] = 65; + Big5PFreq[77][32] = 64; + Big5PFreq[16][1] = 63; + Big5PFreq[25][86] = 62; + Big5PFreq[17][13] = 61; + Big5PFreq[5][75] = 60; + Big5PFreq[36][52] = 59; + Big5PFreq[51][164] = 58; + Big5PFreq[12][85] = 57; + Big5PFreq[39][168] = 56; + Big5PFreq[43][16] = 55; + Big5PFreq[40][69] = 54; + Big5PFreq[26][108] = 53; + Big5PFreq[51][56] = 52; + Big5PFreq[16][37] = 51; + Big5PFreq[40][29] = 50; + Big5PFreq[46][171] = 49; + Big5PFreq[40][128] = 48; + Big5PFreq[72][114] = 47; + Big5PFreq[21][103] = 46; + Big5PFreq[22][44] = 45; + Big5PFreq[40][115] = 44; + Big5PFreq[43][7] = 43; + Big5PFreq[43][153] = 42; + Big5PFreq[17][20] = 41; + Big5PFreq[16][49] = 40; + Big5PFreq[36][57] = 39; + Big5PFreq[18][38] = 38; + Big5PFreq[45][184] = 37; + Big5PFreq[37][167] = 36; + Big5PFreq[26][106] = 35; + Big5PFreq[61][121] = 34; + Big5PFreq[89][140] = 33; + Big5PFreq[46][61] = 32; + Big5PFreq[39][163] = 31; + Big5PFreq[40][62] = 30; + Big5PFreq[38][165] = 29; + Big5PFreq[47][37] = 28; + Big5PFreq[18][155] = 27; + Big5PFreq[20][33] = 26; + Big5PFreq[29][90] = 25; + Big5PFreq[20][103] = 24; + Big5PFreq[37][51] = 23; + Big5PFreq[57][0] = 22; + Big5PFreq[40][31] = 21; + Big5PFreq[45][32] = 20; + Big5PFreq[59][23] = 19; + Big5PFreq[18][47] = 18; + Big5PFreq[45][134] = 17; + Big5PFreq[37][59] = 16; + Big5PFreq[21][128] = 15; + Big5PFreq[36][106] = 14; + Big5PFreq[31][39] = 13; + Big5PFreq[40][182] = 12; + Big5PFreq[52][155] = 11; + Big5PFreq[42][166] = 10; + Big5PFreq[35][27] = 9; + Big5PFreq[38][3] = 8; + Big5PFreq[13][44] = 7; + Big5PFreq[58][157] = 6; + Big5PFreq[47][51] = 5; + Big5PFreq[41][37] = 4; + Big5PFreq[41][172] = 3; + Big5PFreq[51][165] = 2; + Big5PFreq[15][161] = 1; + Big5PFreq[24][181] = 0; + EUC_TWFreq[48][49] = 599; + EUC_TWFreq[35][65] = 598; + EUC_TWFreq[41][27] = 597; + EUC_TWFreq[35][0] = 596; + EUC_TWFreq[39][19] = 595; + EUC_TWFreq[35][42] = 594; + EUC_TWFreq[38][66] = 593; + EUC_TWFreq[35][8] = 592; + EUC_TWFreq[35][6] = 591; + EUC_TWFreq[35][66] = 590; + EUC_TWFreq[43][14] = 589; + EUC_TWFreq[69][80] = 588; + EUC_TWFreq[50][48] = 587; + EUC_TWFreq[36][71] = 586; + EUC_TWFreq[37][10] = 585; + EUC_TWFreq[60][52] = 584; + EUC_TWFreq[51][21] = 583; + EUC_TWFreq[40][2] = 582; + EUC_TWFreq[67][35] = 581; + EUC_TWFreq[38][78] = 580; + EUC_TWFreq[49][18] = 579; + EUC_TWFreq[35][23] = 578; + EUC_TWFreq[42][83] = 577; + EUC_TWFreq[79][47] = 576; + EUC_TWFreq[61][82] = 575; + EUC_TWFreq[38][7] = 574; + EUC_TWFreq[35][29] = 573; + EUC_TWFreq[37][77] = 572; + EUC_TWFreq[54][67] = 571; + EUC_TWFreq[38][80] = 570; + EUC_TWFreq[52][74] = 569; + EUC_TWFreq[36][37] = 568; + EUC_TWFreq[74][8] = 567; + EUC_TWFreq[41][83] = 566; + EUC_TWFreq[36][75] = 565; + EUC_TWFreq[49][63] = 564; + EUC_TWFreq[42][58] = 563; + EUC_TWFreq[56][33] = 562; + EUC_TWFreq[37][76] = 561; + EUC_TWFreq[62][39] = 560; + EUC_TWFreq[35][21] = 559; + EUC_TWFreq[70][19] = 558; + EUC_TWFreq[77][88] = 557; + EUC_TWFreq[51][14] = 556; + EUC_TWFreq[36][17] = 555; + EUC_TWFreq[44][51] = 554; + EUC_TWFreq[38][72] = 553; + EUC_TWFreq[74][90] = 552; + EUC_TWFreq[35][48] = 551; + EUC_TWFreq[35][69] = 550; + EUC_TWFreq[66][86] = 549; + EUC_TWFreq[57][20] = 548; + EUC_TWFreq[35][53] = 547; + EUC_TWFreq[36][87] = 546; + EUC_TWFreq[84][67] = 545; + EUC_TWFreq[70][56] = 544; + EUC_TWFreq[71][54] = 543; + EUC_TWFreq[60][70] = 542; + EUC_TWFreq[80][1] = 541; + EUC_TWFreq[39][59] = 540; + EUC_TWFreq[39][51] = 539; + EUC_TWFreq[35][44] = 538; + EUC_TWFreq[48][4] = 537; + EUC_TWFreq[55][24] = 536; + EUC_TWFreq[52][4] = 535; + EUC_TWFreq[54][26] = 534; + EUC_TWFreq[36][31] = 533; + EUC_TWFreq[37][22] = 532; + EUC_TWFreq[37][9] = 531; + EUC_TWFreq[46][0] = 530; + EUC_TWFreq[56][46] = 529; + EUC_TWFreq[47][93] = 528; + EUC_TWFreq[37][25] = 527; + EUC_TWFreq[39][8] = 526; + EUC_TWFreq[46][73] = 525; + EUC_TWFreq[38][48] = 524; + EUC_TWFreq[39][83] = 523; + EUC_TWFreq[60][92] = 522; + EUC_TWFreq[70][11] = 521; + EUC_TWFreq[63][84] = 520; + EUC_TWFreq[38][65] = 519; + EUC_TWFreq[45][45] = 518; + EUC_TWFreq[63][49] = 517; + EUC_TWFreq[63][50] = 516; + EUC_TWFreq[39][93] = 515; + EUC_TWFreq[68][20] = 514; + EUC_TWFreq[44][84] = 513; + EUC_TWFreq[66][34] = 512; + EUC_TWFreq[37][58] = 511; + EUC_TWFreq[39][0] = 510; + EUC_TWFreq[59][1] = 509; + EUC_TWFreq[47][8] = 508; + EUC_TWFreq[61][17] = 507; + EUC_TWFreq[53][87] = 506; + EUC_TWFreq[67][26] = 505; + EUC_TWFreq[43][46] = 504; + EUC_TWFreq[38][61] = 503; + EUC_TWFreq[45][9] = 502; + EUC_TWFreq[66][83] = 501; + EUC_TWFreq[43][88] = 500; + EUC_TWFreq[85][20] = 499; + EUC_TWFreq[57][36] = 498; + EUC_TWFreq[43][6] = 497; + EUC_TWFreq[86][77] = 496; + EUC_TWFreq[42][70] = 495; + EUC_TWFreq[49][78] = 494; + EUC_TWFreq[36][40] = 493; + EUC_TWFreq[42][71] = 492; + EUC_TWFreq[58][49] = 491; + EUC_TWFreq[35][20] = 490; + EUC_TWFreq[76][20] = 489; + EUC_TWFreq[39][25] = 488; + EUC_TWFreq[40][34] = 487; + EUC_TWFreq[39][76] = 486; + EUC_TWFreq[40][1] = 485; + EUC_TWFreq[59][0] = 484; + EUC_TWFreq[39][70] = 483; + EUC_TWFreq[46][14] = 482; + EUC_TWFreq[68][77] = 481; + EUC_TWFreq[38][55] = 480; + EUC_TWFreq[35][78] = 479; + EUC_TWFreq[84][44] = 478; + EUC_TWFreq[36][41] = 477; + EUC_TWFreq[37][62] = 476; + EUC_TWFreq[65][67] = 475; + EUC_TWFreq[69][66] = 474; + EUC_TWFreq[73][55] = 473; + EUC_TWFreq[71][49] = 472; + EUC_TWFreq[66][87] = 471; + EUC_TWFreq[38][33] = 470; + EUC_TWFreq[64][61] = 469; + EUC_TWFreq[35][7] = 468; + EUC_TWFreq[47][49] = 467; + EUC_TWFreq[56][14] = 466; + EUC_TWFreq[36][49] = 465; + EUC_TWFreq[50][81] = 464; + EUC_TWFreq[55][76] = 463; + EUC_TWFreq[35][19] = 462; + EUC_TWFreq[44][47] = 461; + EUC_TWFreq[35][15] = 460; + EUC_TWFreq[82][59] = 459; + EUC_TWFreq[35][43] = 458; + EUC_TWFreq[73][0] = 457; + EUC_TWFreq[57][83] = 456; + EUC_TWFreq[42][46] = 455; + EUC_TWFreq[36][0] = 454; + EUC_TWFreq[70][88] = 453; + EUC_TWFreq[42][22] = 452; + EUC_TWFreq[46][58] = 451; + EUC_TWFreq[36][34] = 450; + EUC_TWFreq[39][24] = 449; + EUC_TWFreq[35][55] = 448; + EUC_TWFreq[44][91] = 447; + EUC_TWFreq[37][51] = 446; + EUC_TWFreq[36][19] = 445; + EUC_TWFreq[69][90] = 444; + EUC_TWFreq[55][35] = 443; + EUC_TWFreq[35][54] = 442; + EUC_TWFreq[49][61] = 441; + EUC_TWFreq[36][67] = 440; + EUC_TWFreq[88][34] = 439; + EUC_TWFreq[35][17] = 438; + EUC_TWFreq[65][69] = 437; + EUC_TWFreq[74][89] = 436; + EUC_TWFreq[37][31] = 435; + EUC_TWFreq[43][48] = 434; + EUC_TWFreq[89][27] = 433; + EUC_TWFreq[42][79] = 432; + EUC_TWFreq[69][57] = 431; + EUC_TWFreq[36][13] = 430; + EUC_TWFreq[35][62] = 429; + EUC_TWFreq[65][47] = 428; + EUC_TWFreq[56][8] = 427; + EUC_TWFreq[38][79] = 426; + EUC_TWFreq[37][64] = 425; + EUC_TWFreq[64][64] = 424; + EUC_TWFreq[38][53] = 423; + EUC_TWFreq[38][31] = 422; + EUC_TWFreq[56][81] = 421; + EUC_TWFreq[36][22] = 420; + EUC_TWFreq[43][4] = 419; + EUC_TWFreq[36][90] = 418; + EUC_TWFreq[38][62] = 417; + EUC_TWFreq[66][85] = 416; + EUC_TWFreq[39][1] = 415; + EUC_TWFreq[59][40] = 414; + EUC_TWFreq[58][93] = 413; + EUC_TWFreq[44][43] = 412; + EUC_TWFreq[39][49] = 411; + EUC_TWFreq[64][2] = 410; + EUC_TWFreq[41][35] = 409; + EUC_TWFreq[60][22] = 408; + EUC_TWFreq[35][91] = 407; + EUC_TWFreq[78][1] = 406; + EUC_TWFreq[36][14] = 405; + EUC_TWFreq[82][29] = 404; + EUC_TWFreq[52][86] = 403; + EUC_TWFreq[40][16] = 402; + EUC_TWFreq[91][52] = 401; + EUC_TWFreq[50][75] = 400; + EUC_TWFreq[64][30] = 399; + EUC_TWFreq[90][78] = 398; + EUC_TWFreq[36][52] = 397; + EUC_TWFreq[55][87] = 396; + EUC_TWFreq[57][5] = 395; + EUC_TWFreq[57][31] = 394; + EUC_TWFreq[42][35] = 393; + EUC_TWFreq[69][50] = 392; + EUC_TWFreq[45][8] = 391; + EUC_TWFreq[50][87] = 390; + EUC_TWFreq[69][55] = 389; + EUC_TWFreq[92][3] = 388; + EUC_TWFreq[36][43] = 387; + EUC_TWFreq[64][10] = 386; + EUC_TWFreq[56][25] = 385; + EUC_TWFreq[60][68] = 384; + EUC_TWFreq[51][46] = 383; + EUC_TWFreq[50][0] = 382; + EUC_TWFreq[38][30] = 381; + EUC_TWFreq[50][85] = 380; + EUC_TWFreq[60][54] = 379; + EUC_TWFreq[73][6] = 378; + EUC_TWFreq[73][28] = 377; + EUC_TWFreq[56][19] = 376; + EUC_TWFreq[62][69] = 375; + EUC_TWFreq[81][66] = 374; + EUC_TWFreq[40][32] = 373; + EUC_TWFreq[76][31] = 372; + EUC_TWFreq[35][10] = 371; + EUC_TWFreq[41][37] = 370; + EUC_TWFreq[52][82] = 369; + EUC_TWFreq[91][72] = 368; + EUC_TWFreq[37][29] = 367; + EUC_TWFreq[56][30] = 366; + EUC_TWFreq[37][80] = 365; + EUC_TWFreq[81][56] = 364; + EUC_TWFreq[70][3] = 363; + EUC_TWFreq[76][15] = 362; + EUC_TWFreq[46][47] = 361; + EUC_TWFreq[35][88] = 360; + EUC_TWFreq[61][58] = 359; + EUC_TWFreq[37][37] = 358; + EUC_TWFreq[57][22] = 357; + EUC_TWFreq[41][23] = 356; + EUC_TWFreq[90][66] = 355; + EUC_TWFreq[39][60] = 354; + EUC_TWFreq[38][0] = 353; + EUC_TWFreq[37][87] = 352; + EUC_TWFreq[46][2] = 351; + EUC_TWFreq[38][56] = 350; + EUC_TWFreq[58][11] = 349; + EUC_TWFreq[48][10] = 348; + EUC_TWFreq[74][4] = 347; + EUC_TWFreq[40][42] = 346; + EUC_TWFreq[41][52] = 345; + EUC_TWFreq[61][92] = 344; + EUC_TWFreq[39][50] = 343; + EUC_TWFreq[47][88] = 342; + EUC_TWFreq[88][36] = 341; + EUC_TWFreq[45][73] = 340; + EUC_TWFreq[82][3] = 339; + EUC_TWFreq[61][36] = 338; + EUC_TWFreq[60][33] = 337; + EUC_TWFreq[38][27] = 336; + EUC_TWFreq[35][83] = 335; + EUC_TWFreq[65][24] = 334; + EUC_TWFreq[73][10] = 333; + EUC_TWFreq[41][13] = 332; + EUC_TWFreq[50][27] = 331; + EUC_TWFreq[59][50] = 330; + EUC_TWFreq[42][45] = 329; + EUC_TWFreq[55][19] = 328; + EUC_TWFreq[36][77] = 327; + EUC_TWFreq[69][31] = 326; + EUC_TWFreq[60][7] = 325; + EUC_TWFreq[40][88] = 324; + EUC_TWFreq[57][56] = 323; + EUC_TWFreq[50][50] = 322; + EUC_TWFreq[42][37] = 321; + EUC_TWFreq[38][82] = 320; + EUC_TWFreq[52][25] = 319; + EUC_TWFreq[42][67] = 318; + EUC_TWFreq[48][40] = 317; + EUC_TWFreq[45][81] = 316; + EUC_TWFreq[57][14] = 315; + EUC_TWFreq[42][13] = 314; + EUC_TWFreq[78][0] = 313; + EUC_TWFreq[35][51] = 312; + EUC_TWFreq[41][67] = 311; + EUC_TWFreq[64][23] = 310; + EUC_TWFreq[36][65] = 309; + EUC_TWFreq[48][50] = 308; + EUC_TWFreq[46][69] = 307; + EUC_TWFreq[47][89] = 306; + EUC_TWFreq[41][48] = 305; + EUC_TWFreq[60][56] = 304; + EUC_TWFreq[44][82] = 303; + EUC_TWFreq[47][35] = 302; + EUC_TWFreq[49][3] = 301; + EUC_TWFreq[49][69] = 300; + EUC_TWFreq[45][93] = 299; + EUC_TWFreq[60][34] = 298; + EUC_TWFreq[60][82] = 297; + EUC_TWFreq[61][61] = 296; + EUC_TWFreq[86][42] = 295; + EUC_TWFreq[89][60] = 294; + EUC_TWFreq[48][31] = 293; + EUC_TWFreq[35][75] = 292; + EUC_TWFreq[91][39] = 291; + EUC_TWFreq[53][19] = 290; + EUC_TWFreq[39][72] = 289; + EUC_TWFreq[69][59] = 288; + EUC_TWFreq[41][7] = 287; + EUC_TWFreq[54][13] = 286; + EUC_TWFreq[43][28] = 285; + EUC_TWFreq[36][6] = 284; + EUC_TWFreq[45][75] = 283; + EUC_TWFreq[36][61] = 282; + EUC_TWFreq[38][21] = 281; + EUC_TWFreq[45][14] = 280; + EUC_TWFreq[61][43] = 279; + EUC_TWFreq[36][63] = 278; + EUC_TWFreq[43][30] = 277; + EUC_TWFreq[46][51] = 276; + EUC_TWFreq[68][87] = 275; + EUC_TWFreq[39][26] = 274; + EUC_TWFreq[46][76] = 273; + EUC_TWFreq[36][15] = 272; + EUC_TWFreq[35][40] = 271; + EUC_TWFreq[79][60] = 270; + EUC_TWFreq[46][7] = 269; + EUC_TWFreq[65][72] = 268; + EUC_TWFreq[69][88] = 267; + EUC_TWFreq[47][18] = 266; + EUC_TWFreq[37][0] = 265; + EUC_TWFreq[37][49] = 264; + EUC_TWFreq[67][37] = 263; + EUC_TWFreq[36][91] = 262; + EUC_TWFreq[75][48] = 261; + EUC_TWFreq[75][63] = 260; + EUC_TWFreq[83][87] = 259; + EUC_TWFreq[37][44] = 258; + EUC_TWFreq[73][54] = 257; + EUC_TWFreq[51][61] = 256; + EUC_TWFreq[46][57] = 255; + EUC_TWFreq[55][21] = 254; + EUC_TWFreq[39][66] = 253; + EUC_TWFreq[47][11] = 252; + EUC_TWFreq[52][8] = 251; + EUC_TWFreq[82][81] = 250; + EUC_TWFreq[36][57] = 249; + EUC_TWFreq[38][54] = 248; + EUC_TWFreq[43][81] = 247; + EUC_TWFreq[37][42] = 246; + EUC_TWFreq[40][18] = 245; + EUC_TWFreq[80][90] = 244; + EUC_TWFreq[37][84] = 243; + EUC_TWFreq[57][15] = 242; + EUC_TWFreq[38][87] = 241; + EUC_TWFreq[37][32] = 240; + EUC_TWFreq[53][53] = 239; + EUC_TWFreq[89][29] = 238; + EUC_TWFreq[81][53] = 237; + EUC_TWFreq[75][3] = 236; + EUC_TWFreq[83][73] = 235; + EUC_TWFreq[66][13] = 234; + EUC_TWFreq[48][7] = 233; + EUC_TWFreq[46][35] = 232; + EUC_TWFreq[35][86] = 231; + EUC_TWFreq[37][20] = 230; + EUC_TWFreq[46][80] = 229; + EUC_TWFreq[38][24] = 228; + EUC_TWFreq[41][68] = 227; + EUC_TWFreq[42][21] = 226; + EUC_TWFreq[43][32] = 225; + EUC_TWFreq[38][20] = 224; + EUC_TWFreq[37][59] = 223; + EUC_TWFreq[41][77] = 222; + EUC_TWFreq[59][57] = 221; + EUC_TWFreq[68][59] = 220; + EUC_TWFreq[39][43] = 219; + EUC_TWFreq[54][39] = 218; + EUC_TWFreq[48][28] = 217; + EUC_TWFreq[54][28] = 216; + EUC_TWFreq[41][44] = 215; + EUC_TWFreq[51][64] = 214; + EUC_TWFreq[47][72] = 213; + EUC_TWFreq[62][67] = 212; + EUC_TWFreq[42][43] = 211; + EUC_TWFreq[61][38] = 210; + EUC_TWFreq[76][25] = 209; + EUC_TWFreq[48][91] = 208; + EUC_TWFreq[36][36] = 207; + EUC_TWFreq[80][32] = 206; + EUC_TWFreq[81][40] = 205; + EUC_TWFreq[37][5] = 204; + EUC_TWFreq[74][69] = 203; + EUC_TWFreq[36][82] = 202; + EUC_TWFreq[46][59] = 201; + /* + * EUC_TWFreq[38][32] = 200; EUC_TWFreq[74][2] = 199; EUC_TWFreq[53][31] = 198; EUC_TWFreq[35][38] = 197; EUC_TWFreq[46][62] = + * 196; EUC_TWFreq[77][31] = 195; EUC_TWFreq[55][74] = 194; EUC_TWFreq[66][6] = 193; EUC_TWFreq[56][21] = 192; + * EUC_TWFreq[54][78] = 191; EUC_TWFreq[43][51] = 190; EUC_TWFreq[64][93] = 189; EUC_TWFreq[92][7] = 188; EUC_TWFreq[83][89] = + * 187; EUC_TWFreq[69][9] = 186; EUC_TWFreq[45][4] = 185; EUC_TWFreq[53][9] = 184; EUC_TWFreq[43][2] = 183; + * EUC_TWFreq[35][11] = 182; EUC_TWFreq[51][25] = 181; EUC_TWFreq[52][71] = 180; EUC_TWFreq[81][67] = 179; + * EUC_TWFreq[37][33] = 178; EUC_TWFreq[38][57] = 177; EUC_TWFreq[39][77] = 176; EUC_TWFreq[40][26] = 175; + * EUC_TWFreq[37][21] = 174; EUC_TWFreq[81][70] = 173; EUC_TWFreq[56][80] = 172; EUC_TWFreq[65][14] = 171; + * EUC_TWFreq[62][47] = 170; EUC_TWFreq[56][54] = 169; EUC_TWFreq[45][17] = 168; EUC_TWFreq[52][52] = 167; + * EUC_TWFreq[74][30] = 166; EUC_TWFreq[60][57] = 165; EUC_TWFreq[41][15] = 164; EUC_TWFreq[47][69] = 163; + * EUC_TWFreq[61][11] = 162; EUC_TWFreq[72][25] = 161; EUC_TWFreq[82][56] = 160; EUC_TWFreq[76][92] = 159; + * EUC_TWFreq[51][22] = 158; EUC_TWFreq[55][69] = 157; EUC_TWFreq[49][43] = 156; EUC_TWFreq[69][49] = 155; + * EUC_TWFreq[88][42] = 154; EUC_TWFreq[84][41] = 153; EUC_TWFreq[79][33] = 152; EUC_TWFreq[47][17] = 151; + * EUC_TWFreq[52][88] = 150; EUC_TWFreq[63][74] = 149; EUC_TWFreq[50][32] = 148; EUC_TWFreq[65][10] = 147; EUC_TWFreq[57][6] = + * 146; EUC_TWFreq[52][23] = 145; EUC_TWFreq[36][70] = 144; EUC_TWFreq[65][55] = 143; EUC_TWFreq[35][27] = 142; + * EUC_TWFreq[57][63] = 141; EUC_TWFreq[39][92] = 140; EUC_TWFreq[79][75] = 139; EUC_TWFreq[36][30] = 138; + * EUC_TWFreq[53][60] = 137; EUC_TWFreq[55][43] = 136; EUC_TWFreq[71][22] = 135; EUC_TWFreq[43][16] = 134; + * EUC_TWFreq[65][21] = 133; EUC_TWFreq[84][51] = 132; EUC_TWFreq[43][64] = 131; EUC_TWFreq[87][91] = 130; + * EUC_TWFreq[47][45] = 129; EUC_TWFreq[65][29] = 128; EUC_TWFreq[88][16] = 127; EUC_TWFreq[50][5] = 126; EUC_TWFreq[47][33] = + * 125; EUC_TWFreq[46][27] = 124; EUC_TWFreq[85][2] = 123; EUC_TWFreq[43][77] = 122; EUC_TWFreq[70][9] = 121; + * EUC_TWFreq[41][54] = 120; EUC_TWFreq[56][12] = 119; EUC_TWFreq[90][65] = 118; EUC_TWFreq[91][50] = 117; + * EUC_TWFreq[48][41] = 116; EUC_TWFreq[35][89] = 115; EUC_TWFreq[90][83] = 114; EUC_TWFreq[44][40] = 113; + * EUC_TWFreq[50][88] = 112; EUC_TWFreq[72][39] = 111; EUC_TWFreq[45][3] = 110; EUC_TWFreq[71][33] = 109; EUC_TWFreq[39][12] = + * 108; EUC_TWFreq[59][24] = 107; EUC_TWFreq[60][62] = 106; EUC_TWFreq[44][33] = 105; EUC_TWFreq[53][70] = 104; + * EUC_TWFreq[77][90] = 103; EUC_TWFreq[50][58] = 102; EUC_TWFreq[54][1] = 101; EUC_TWFreq[73][19] = 100; EUC_TWFreq[37][3] = + * 99; EUC_TWFreq[49][91] = 98; EUC_TWFreq[88][43] = 97; EUC_TWFreq[36][78] = 96; EUC_TWFreq[44][20] = 95; + * EUC_TWFreq[64][15] = 94; EUC_TWFreq[72][28] = 93; EUC_TWFreq[70][13] = 92; EUC_TWFreq[65][83] = 91; EUC_TWFreq[58][68] = + * 90; EUC_TWFreq[59][32] = 89; EUC_TWFreq[39][13] = 88; EUC_TWFreq[55][64] = 87; EUC_TWFreq[56][59] = 86; + * EUC_TWFreq[39][17] = 85; EUC_TWFreq[55][84] = 84; EUC_TWFreq[77][85] = 83; EUC_TWFreq[60][19] = 82; EUC_TWFreq[62][82] = + * 81; EUC_TWFreq[78][16] = 80; EUC_TWFreq[66][8] = 79; EUC_TWFreq[39][42] = 78; EUC_TWFreq[61][24] = 77; EUC_TWFreq[57][67] = + * 76; EUC_TWFreq[38][83] = 75; EUC_TWFreq[36][53] = 74; EUC_TWFreq[67][76] = 73; EUC_TWFreq[37][91] = 72; + * EUC_TWFreq[44][26] = 71; EUC_TWFreq[72][86] = 70; EUC_TWFreq[44][87] = 69; EUC_TWFreq[45][50] = 68; EUC_TWFreq[58][4] = + * 67; EUC_TWFreq[86][65] = 66; EUC_TWFreq[45][56] = 65; EUC_TWFreq[79][49] = 64; EUC_TWFreq[35][3] = 63; EUC_TWFreq[48][83] = + * 62; EUC_TWFreq[71][21] = 61; EUC_TWFreq[77][93] = 60; EUC_TWFreq[87][92] = 59; EUC_TWFreq[38][35] = 58; + * EUC_TWFreq[66][17] = 57; EUC_TWFreq[37][66] = 56; EUC_TWFreq[51][42] = 55; EUC_TWFreq[57][73] = 54; EUC_TWFreq[51][54] = + * 53; EUC_TWFreq[75][64] = 52; EUC_TWFreq[35][5] = 51; EUC_TWFreq[49][40] = 50; EUC_TWFreq[58][35] = 49; EUC_TWFreq[67][88] = + * 48; EUC_TWFreq[60][51] = 47; EUC_TWFreq[36][92] = 46; EUC_TWFreq[44][41] = 45; EUC_TWFreq[58][29] = 44; + * EUC_TWFreq[43][62] = 43; EUC_TWFreq[56][23] = 42; EUC_TWFreq[67][44] = 41; EUC_TWFreq[52][91] = 40; EUC_TWFreq[42][81] = + * 39; EUC_TWFreq[64][25] = 38; EUC_TWFreq[35][36] = 37; EUC_TWFreq[47][73] = 36; EUC_TWFreq[36][1] = 35; EUC_TWFreq[65][84] = + * 34; EUC_TWFreq[73][1] = 33; EUC_TWFreq[79][66] = 32; EUC_TWFreq[69][14] = 31; EUC_TWFreq[65][28] = 30; EUC_TWFreq[60][93] = + * 29; EUC_TWFreq[72][79] = 28; EUC_TWFreq[48][0] = 27; EUC_TWFreq[73][43] = 26; EUC_TWFreq[66][47] = 25; EUC_TWFreq[41][18] = + * 24; EUC_TWFreq[51][10] = 23; EUC_TWFreq[59][7] = 22; EUC_TWFreq[53][27] = 21; EUC_TWFreq[86][67] = 20; EUC_TWFreq[49][87] = + * 19; EUC_TWFreq[52][28] = 18; EUC_TWFreq[52][12] = 17; EUC_TWFreq[42][30] = 16; EUC_TWFreq[65][35] = 15; + * EUC_TWFreq[46][64] = 14; EUC_TWFreq[71][7] = 13; EUC_TWFreq[56][57] = 12; EUC_TWFreq[56][31] = 11; EUC_TWFreq[41][31] = + * 10; EUC_TWFreq[48][59] = 9; EUC_TWFreq[63][92] = 8; EUC_TWFreq[62][57] = 7; EUC_TWFreq[65][87] = 6; EUC_TWFreq[70][10] = + * 5; EUC_TWFreq[52][40] = 4; EUC_TWFreq[40][22] = 3; EUC_TWFreq[65][91] = 2; EUC_TWFreq[50][25] = 1; EUC_TWFreq[35][84] = + * 0; + */ + GBKFreq[52][132] = 600; + GBKFreq[73][135] = 599; + GBKFreq[49][123] = 598; + GBKFreq[77][146] = 597; + GBKFreq[81][123] = 596; + GBKFreq[82][144] = 595; + GBKFreq[51][179] = 594; + GBKFreq[83][154] = 593; + GBKFreq[71][139] = 592; + GBKFreq[64][139] = 591; + GBKFreq[85][144] = 590; + GBKFreq[52][125] = 589; + GBKFreq[88][25] = 588; + GBKFreq[81][106] = 587; + GBKFreq[81][148] = 586; + GBKFreq[62][137] = 585; + GBKFreq[94][0] = 584; + GBKFreq[1][64] = 583; + GBKFreq[67][163] = 582; + GBKFreq[20][190] = 581; + GBKFreq[57][131] = 580; + GBKFreq[29][169] = 579; + GBKFreq[72][143] = 578; + GBKFreq[0][173] = 577; + GBKFreq[11][23] = 576; + GBKFreq[61][141] = 575; + GBKFreq[60][123] = 574; + GBKFreq[81][114] = 573; + GBKFreq[82][131] = 572; + GBKFreq[67][156] = 571; + GBKFreq[71][167] = 570; + GBKFreq[20][50] = 569; + GBKFreq[77][132] = 568; + GBKFreq[84][38] = 567; + GBKFreq[26][29] = 566; + GBKFreq[74][187] = 565; + GBKFreq[62][116] = 564; + GBKFreq[67][135] = 563; + GBKFreq[5][86] = 562; + GBKFreq[72][186] = 561; + GBKFreq[75][161] = 560; + GBKFreq[78][130] = 559; + GBKFreq[94][30] = 558; + GBKFreq[84][72] = 557; + GBKFreq[1][67] = 556; + GBKFreq[75][172] = 555; + GBKFreq[74][185] = 554; + GBKFreq[53][160] = 553; + GBKFreq[123][14] = 552; + GBKFreq[79][97] = 551; + GBKFreq[85][110] = 550; + GBKFreq[78][171] = 549; + GBKFreq[52][131] = 548; + GBKFreq[56][100] = 547; + GBKFreq[50][182] = 546; + GBKFreq[94][64] = 545; + GBKFreq[106][74] = 544; + GBKFreq[11][102] = 543; + GBKFreq[53][124] = 542; + GBKFreq[24][3] = 541; + GBKFreq[86][148] = 540; + GBKFreq[53][184] = 539; + GBKFreq[86][147] = 538; + GBKFreq[96][161] = 537; + GBKFreq[82][77] = 536; + GBKFreq[59][146] = 535; + GBKFreq[84][126] = 534; + GBKFreq[79][132] = 533; + GBKFreq[85][123] = 532; + GBKFreq[71][101] = 531; + GBKFreq[85][106] = 530; + GBKFreq[6][184] = 529; + GBKFreq[57][156] = 528; + GBKFreq[75][104] = 527; + GBKFreq[50][137] = 526; + GBKFreq[79][133] = 525; + GBKFreq[76][108] = 524; + GBKFreq[57][142] = 523; + GBKFreq[84][130] = 522; + GBKFreq[52][128] = 521; + GBKFreq[47][44] = 520; + GBKFreq[52][152] = 519; + GBKFreq[54][104] = 518; + GBKFreq[30][47] = 517; + GBKFreq[71][123] = 516; + GBKFreq[52][107] = 515; + GBKFreq[45][84] = 514; + GBKFreq[107][118] = 513; + GBKFreq[5][161] = 512; + GBKFreq[48][126] = 511; + GBKFreq[67][170] = 510; + GBKFreq[43][6] = 509; + GBKFreq[70][112] = 508; + GBKFreq[86][174] = 507; + GBKFreq[84][166] = 506; + GBKFreq[79][130] = 505; + GBKFreq[57][141] = 504; + GBKFreq[81][178] = 503; + GBKFreq[56][187] = 502; + GBKFreq[81][162] = 501; + GBKFreq[53][104] = 500; + GBKFreq[123][35] = 499; + GBKFreq[70][169] = 498; + GBKFreq[69][164] = 497; + GBKFreq[109][61] = 496; + GBKFreq[73][130] = 495; + GBKFreq[62][134] = 494; + GBKFreq[54][125] = 493; + GBKFreq[79][105] = 492; + GBKFreq[70][165] = 491; + GBKFreq[71][189] = 490; + GBKFreq[23][147] = 489; + GBKFreq[51][139] = 488; + GBKFreq[47][137] = 487; + GBKFreq[77][123] = 486; + GBKFreq[86][183] = 485; + GBKFreq[63][173] = 484; + GBKFreq[79][144] = 483; + GBKFreq[84][159] = 482; + GBKFreq[60][91] = 481; + GBKFreq[66][187] = 480; + GBKFreq[73][114] = 479; + GBKFreq[85][56] = 478; + GBKFreq[71][149] = 477; + GBKFreq[84][189] = 476; + GBKFreq[104][31] = 475; + GBKFreq[83][82] = 474; + GBKFreq[68][35] = 473; + GBKFreq[11][77] = 472; + GBKFreq[15][155] = 471; + GBKFreq[83][153] = 470; + GBKFreq[71][1] = 469; + GBKFreq[53][190] = 468; + GBKFreq[50][135] = 467; + GBKFreq[3][147] = 466; + GBKFreq[48][136] = 465; + GBKFreq[66][166] = 464; + GBKFreq[55][159] = 463; + GBKFreq[82][150] = 462; + GBKFreq[58][178] = 461; + GBKFreq[64][102] = 460; + GBKFreq[16][106] = 459; + GBKFreq[68][110] = 458; + GBKFreq[54][14] = 457; + GBKFreq[60][140] = 456; + GBKFreq[91][71] = 455; + GBKFreq[54][150] = 454; + GBKFreq[78][177] = 453; + GBKFreq[78][117] = 452; + GBKFreq[104][12] = 451; + GBKFreq[73][150] = 450; + GBKFreq[51][142] = 449; + GBKFreq[81][145] = 448; + GBKFreq[66][183] = 447; + GBKFreq[51][178] = 446; + GBKFreq[75][107] = 445; + GBKFreq[65][119] = 444; + GBKFreq[69][176] = 443; + GBKFreq[59][122] = 442; + GBKFreq[78][160] = 441; + GBKFreq[85][183] = 440; + GBKFreq[105][16] = 439; + GBKFreq[73][110] = 438; + GBKFreq[104][39] = 437; + GBKFreq[119][16] = 436; + GBKFreq[76][162] = 435; + GBKFreq[67][152] = 434; + GBKFreq[82][24] = 433; + GBKFreq[73][121] = 432; + GBKFreq[83][83] = 431; + GBKFreq[82][145] = 430; + GBKFreq[49][133] = 429; + GBKFreq[94][13] = 428; + GBKFreq[58][139] = 427; + GBKFreq[74][189] = 426; + GBKFreq[66][177] = 425; + GBKFreq[85][184] = 424; + GBKFreq[55][183] = 423; + GBKFreq[71][107] = 422; + GBKFreq[11][98] = 421; + GBKFreq[72][153] = 420; + GBKFreq[2][137] = 419; + GBKFreq[59][147] = 418; + GBKFreq[58][152] = 417; + GBKFreq[55][144] = 416; + GBKFreq[73][125] = 415; + GBKFreq[52][154] = 414; + GBKFreq[70][178] = 413; + GBKFreq[79][148] = 412; + GBKFreq[63][143] = 411; + GBKFreq[50][140] = 410; + GBKFreq[47][145] = 409; + GBKFreq[48][123] = 408; + GBKFreq[56][107] = 407; + GBKFreq[84][83] = 406; + GBKFreq[59][112] = 405; + GBKFreq[124][72] = 404; + GBKFreq[79][99] = 403; + GBKFreq[3][37] = 402; + GBKFreq[114][55] = 401; + GBKFreq[85][152] = 400; + GBKFreq[60][47] = 399; + GBKFreq[65][96] = 398; + GBKFreq[74][110] = 397; + GBKFreq[86][182] = 396; + GBKFreq[50][99] = 395; + GBKFreq[67][186] = 394; + GBKFreq[81][74] = 393; + GBKFreq[80][37] = 392; + GBKFreq[21][60] = 391; + GBKFreq[110][12] = 390; + GBKFreq[60][162] = 389; + GBKFreq[29][115] = 388; + GBKFreq[83][130] = 387; + GBKFreq[52][136] = 386; + GBKFreq[63][114] = 385; + GBKFreq[49][127] = 384; + GBKFreq[83][109] = 383; + GBKFreq[66][128] = 382; + GBKFreq[78][136] = 381; + GBKFreq[81][180] = 380; + GBKFreq[76][104] = 379; + GBKFreq[56][156] = 378; + GBKFreq[61][23] = 377; + GBKFreq[4][30] = 376; + GBKFreq[69][154] = 375; + GBKFreq[100][37] = 374; + GBKFreq[54][177] = 373; + GBKFreq[23][119] = 372; + GBKFreq[71][171] = 371; + GBKFreq[84][146] = 370; + GBKFreq[20][184] = 369; + GBKFreq[86][76] = 368; + GBKFreq[74][132] = 367; + GBKFreq[47][97] = 366; + GBKFreq[82][137] = 365; + GBKFreq[94][56] = 364; + GBKFreq[92][30] = 363; + GBKFreq[19][117] = 362; + GBKFreq[48][173] = 361; + GBKFreq[2][136] = 360; + GBKFreq[7][182] = 359; + GBKFreq[74][188] = 358; + GBKFreq[14][132] = 357; + GBKFreq[62][172] = 356; + GBKFreq[25][39] = 355; + GBKFreq[85][129] = 354; + GBKFreq[64][98] = 353; + GBKFreq[67][127] = 352; + GBKFreq[72][167] = 351; + GBKFreq[57][143] = 350; + GBKFreq[76][187] = 349; + GBKFreq[83][181] = 348; + GBKFreq[84][10] = 347; + GBKFreq[55][166] = 346; + GBKFreq[55][188] = 345; + GBKFreq[13][151] = 344; + GBKFreq[62][124] = 343; + GBKFreq[53][136] = 342; + GBKFreq[106][57] = 341; + GBKFreq[47][166] = 340; + GBKFreq[109][30] = 339; + GBKFreq[78][114] = 338; + GBKFreq[83][19] = 337; + GBKFreq[56][162] = 336; + GBKFreq[60][177] = 335; + GBKFreq[88][9] = 334; + GBKFreq[74][163] = 333; + GBKFreq[52][156] = 332; + GBKFreq[71][180] = 331; + GBKFreq[60][57] = 330; + GBKFreq[72][173] = 329; + GBKFreq[82][91] = 328; + GBKFreq[51][186] = 327; + GBKFreq[75][86] = 326; + GBKFreq[75][78] = 325; + GBKFreq[76][170] = 324; + GBKFreq[60][147] = 323; + GBKFreq[82][75] = 322; + GBKFreq[80][148] = 321; + GBKFreq[86][150] = 320; + GBKFreq[13][95] = 319; + GBKFreq[0][11] = 318; + GBKFreq[84][190] = 317; + GBKFreq[76][166] = 316; + GBKFreq[14][72] = 315; + GBKFreq[67][144] = 314; + GBKFreq[84][44] = 313; + GBKFreq[72][125] = 312; + GBKFreq[66][127] = 311; + GBKFreq[60][25] = 310; + GBKFreq[70][146] = 309; + GBKFreq[79][135] = 308; + GBKFreq[54][135] = 307; + GBKFreq[60][104] = 306; + GBKFreq[55][132] = 305; + GBKFreq[94][2] = 304; + GBKFreq[54][133] = 303; + GBKFreq[56][190] = 302; + GBKFreq[58][174] = 301; + GBKFreq[80][144] = 300; + GBKFreq[85][113] = 299; + /* + * GBKFreq[83][15] = 298; GBKFreq[105][80] = 297; GBKFreq[7][179] = 296; GBKFreq[93][4] = 295; GBKFreq[123][40] = 294; + * GBKFreq[85][120] = 293; GBKFreq[77][165] = 292; GBKFreq[86][67] = 291; GBKFreq[25][162] = 290; GBKFreq[77][183] = 289; + * GBKFreq[83][71] = 288; GBKFreq[78][99] = 287; GBKFreq[72][177] = 286; GBKFreq[71][97] = 285; GBKFreq[58][111] = 284; + * GBKFreq[77][175] = 283; GBKFreq[76][181] = 282; GBKFreq[71][142] = 281; GBKFreq[64][150] = 280; GBKFreq[5][142] = 279; + * GBKFreq[73][128] = 278; GBKFreq[73][156] = 277; GBKFreq[60][188] = 276; GBKFreq[64][56] = 275; GBKFreq[74][128] = 274; + * GBKFreq[48][163] = 273; GBKFreq[54][116] = 272; GBKFreq[73][127] = 271; GBKFreq[16][176] = 270; GBKFreq[62][149] = 269; + * GBKFreq[105][96] = 268; GBKFreq[55][186] = 267; GBKFreq[4][51] = 266; GBKFreq[48][113] = 265; GBKFreq[48][152] = 264; + * GBKFreq[23][9] = 263; GBKFreq[56][102] = 262; GBKFreq[11][81] = 261; GBKFreq[82][112] = 260; GBKFreq[65][85] = 259; + * GBKFreq[69][125] = 258; GBKFreq[68][31] = 257; GBKFreq[5][20] = 256; GBKFreq[60][176] = 255; GBKFreq[82][81] = 254; + * GBKFreq[72][107] = 253; GBKFreq[3][52] = 252; GBKFreq[71][157] = 251; GBKFreq[24][46] = 250; GBKFreq[69][108] = 249; + * GBKFreq[78][178] = 248; GBKFreq[9][69] = 247; GBKFreq[73][144] = 246; GBKFreq[63][187] = 245; GBKFreq[68][36] = 244; + * GBKFreq[47][151] = 243; GBKFreq[14][74] = 242; GBKFreq[47][114] = 241; GBKFreq[80][171] = 240; GBKFreq[75][152] = 239; + * GBKFreq[86][40] = 238; GBKFreq[93][43] = 237; GBKFreq[2][50] = 236; GBKFreq[62][66] = 235; GBKFreq[1][183] = 234; + * GBKFreq[74][124] = 233; GBKFreq[58][104] = 232; GBKFreq[83][106] = 231; GBKFreq[60][144] = 230; GBKFreq[48][99] = 229; + * GBKFreq[54][157] = 228; GBKFreq[70][179] = 227; GBKFreq[61][127] = 226; GBKFreq[57][135] = 225; GBKFreq[59][190] = 224; + * GBKFreq[77][116] = 223; GBKFreq[26][17] = 222; GBKFreq[60][13] = 221; GBKFreq[71][38] = 220; GBKFreq[85][177] = 219; + * GBKFreq[59][73] = 218; GBKFreq[50][150] = 217; GBKFreq[79][102] = 216; GBKFreq[76][118] = 215; GBKFreq[67][132] = 214; + * GBKFreq[73][146] = 213; GBKFreq[83][184] = 212; GBKFreq[86][159] = 211; GBKFreq[95][120] = 210; GBKFreq[23][139] = 209; + * GBKFreq[64][183] = 208; GBKFreq[85][103] = 207; GBKFreq[41][90] = 206; GBKFreq[87][72] = 205; GBKFreq[62][104] = 204; + * GBKFreq[79][168] = 203; GBKFreq[79][150] = 202; GBKFreq[104][20] = 201; GBKFreq[56][114] = 200; GBKFreq[84][26] = 199; + * GBKFreq[57][99] = 198; GBKFreq[62][154] = 197; GBKFreq[47][98] = 196; GBKFreq[61][64] = 195; GBKFreq[112][18] = 194; + * GBKFreq[123][19] = 193; GBKFreq[4][98] = 192; GBKFreq[47][163] = 191; GBKFreq[66][188] = 190; GBKFreq[81][85] = 189; + * GBKFreq[82][30] = 188; GBKFreq[65][83] = 187; GBKFreq[67][24] = 186; GBKFreq[68][179] = 185; GBKFreq[55][177] = 184; + * GBKFreq[2][122] = 183; GBKFreq[47][139] = 182; GBKFreq[79][158] = 181; GBKFreq[64][143] = 180; GBKFreq[100][24] = 179; + * GBKFreq[73][103] = 178; GBKFreq[50][148] = 177; GBKFreq[86][97] = 176; GBKFreq[59][116] = 175; GBKFreq[64][173] = 174; + * GBKFreq[99][91] = 173; GBKFreq[11][99] = 172; GBKFreq[78][179] = 171; GBKFreq[18][17] = 170; GBKFreq[58][185] = 169; + * GBKFreq[47][165] = 168; GBKFreq[67][131] = 167; GBKFreq[94][40] = 166; GBKFreq[74][153] = 165; GBKFreq[79][142] = 164; + * GBKFreq[57][98] = 163; GBKFreq[1][164] = 162; GBKFreq[55][168] = 161; GBKFreq[13][141] = 160; GBKFreq[51][31] = 159; + * GBKFreq[57][178] = 158; GBKFreq[50][189] = 157; GBKFreq[60][167] = 156; GBKFreq[80][34] = 155; GBKFreq[109][80] = 154; + * GBKFreq[85][54] = 153; GBKFreq[69][183] = 152; GBKFreq[67][143] = 151; GBKFreq[47][120] = 150; GBKFreq[45][75] = 149; + * GBKFreq[82][98] = 148; GBKFreq[83][22] = 147; GBKFreq[13][103] = 146; GBKFreq[49][174] = 145; GBKFreq[57][181] = 144; + * GBKFreq[64][127] = 143; GBKFreq[61][131] = 142; GBKFreq[52][180] = 141; GBKFreq[74][134] = 140; GBKFreq[84][187] = 139; + * GBKFreq[81][189] = 138; GBKFreq[47][160] = 137; GBKFreq[66][148] = 136; GBKFreq[7][4] = 135; GBKFreq[85][134] = 134; + * GBKFreq[88][13] = 133; GBKFreq[88][80] = 132; GBKFreq[69][166] = 131; GBKFreq[86][18] = 130; GBKFreq[79][141] = 129; + * GBKFreq[50][108] = 128; GBKFreq[94][69] = 127; GBKFreq[81][110] = 126; GBKFreq[69][119] = 125; GBKFreq[72][161] = 124; + * GBKFreq[106][45] = 123; GBKFreq[73][124] = 122; GBKFreq[94][28] = 121; GBKFreq[63][174] = 120; GBKFreq[3][149] = 119; + * GBKFreq[24][160] = 118; GBKFreq[113][94] = 117; GBKFreq[56][138] = 116; GBKFreq[64][185] = 115; GBKFreq[86][56] = 114; + * GBKFreq[56][150] = 113; GBKFreq[110][55] = 112; GBKFreq[28][13] = 111; GBKFreq[54][190] = 110; GBKFreq[8][180] = 109; + * GBKFreq[73][149] = 108; GBKFreq[80][155] = 107; GBKFreq[83][172] = 106; GBKFreq[67][174] = 105; GBKFreq[64][180] = 104; + * GBKFreq[84][46] = 103; GBKFreq[91][74] = 102; GBKFreq[69][134] = 101; GBKFreq[61][107] = 100; GBKFreq[47][171] = 99; + * GBKFreq[59][51] = 98; GBKFreq[109][74] = 97; GBKFreq[64][174] = 96; GBKFreq[52][151] = 95; GBKFreq[51][176] = 94; + * GBKFreq[80][157] = 93; GBKFreq[94][31] = 92; GBKFreq[79][155] = 91; GBKFreq[72][174] = 90; GBKFreq[69][113] = 89; + * GBKFreq[83][167] = 88; GBKFreq[83][122] = 87; GBKFreq[8][178] = 86; GBKFreq[70][186] = 85; GBKFreq[59][153] = 84; + * GBKFreq[84][68] = 83; GBKFreq[79][39] = 82; GBKFreq[47][180] = 81; GBKFreq[88][53] = 80; GBKFreq[57][154] = 79; + * GBKFreq[47][153] = 78; GBKFreq[3][153] = 77; GBKFreq[76][134] = 76; GBKFreq[51][166] = 75; GBKFreq[58][176] = 74; + * GBKFreq[27][138] = 73; GBKFreq[73][126] = 72; GBKFreq[76][185] = 71; GBKFreq[52][186] = 70; GBKFreq[81][151] = 69; + * GBKFreq[26][50] = 68; GBKFreq[76][173] = 67; GBKFreq[106][56] = 66; GBKFreq[85][142] = 65; GBKFreq[11][103] = 64; + * GBKFreq[69][159] = 63; GBKFreq[53][142] = 62; GBKFreq[7][6] = 61; GBKFreq[84][59] = 60; GBKFreq[86][3] = 59; + * GBKFreq[64][144] = 58; GBKFreq[1][187] = 57; GBKFreq[82][128] = 56; GBKFreq[3][66] = 55; GBKFreq[68][133] = 54; + * GBKFreq[55][167] = 53; GBKFreq[52][130] = 52; GBKFreq[61][133] = 51; GBKFreq[72][181] = 50; GBKFreq[25][98] = 49; + * GBKFreq[84][149] = 48; GBKFreq[91][91] = 47; GBKFreq[47][188] = 46; GBKFreq[68][130] = 45; GBKFreq[22][44] = 44; + * GBKFreq[81][121] = 43; GBKFreq[72][140] = 42; GBKFreq[55][133] = 41; GBKFreq[55][185] = 40; GBKFreq[56][105] = 39; + * GBKFreq[60][30] = 38; GBKFreq[70][103] = 37; GBKFreq[62][141] = 36; GBKFreq[70][144] = 35; GBKFreq[59][111] = 34; + * GBKFreq[54][17] = 33; GBKFreq[18][190] = 32; GBKFreq[65][164] = 31; GBKFreq[83][125] = 30; GBKFreq[61][121] = 29; + * GBKFreq[48][13] = 28; GBKFreq[51][189] = 27; GBKFreq[65][68] = 26; GBKFreq[7][0] = 25; GBKFreq[76][188] = 24; + * GBKFreq[85][117] = 23; GBKFreq[45][33] = 22; GBKFreq[78][187] = 21; GBKFreq[106][48] = 20; GBKFreq[59][52] = 19; + * GBKFreq[86][185] = 18; GBKFreq[84][121] = 17; GBKFreq[82][189] = 16; GBKFreq[68][156] = 15; GBKFreq[55][125] = 14; + * GBKFreq[65][175] = 13; GBKFreq[7][140] = 12; GBKFreq[50][106] = 11; GBKFreq[59][124] = 10; GBKFreq[67][115] = 9; + * GBKFreq[82][114] = 8; GBKFreq[74][121] = 7; GBKFreq[106][69] = 6; GBKFreq[94][27] = 5; GBKFreq[78][98] = 4; + * GBKFreq[85][186] = 3; GBKFreq[108][90] = 2; GBKFreq[62][160] = 1; GBKFreq[60][169] = 0; + */ + KRFreq[31][43] = 600; + KRFreq[19][56] = 599; + KRFreq[38][46] = 598; + KRFreq[3][3] = 597; + KRFreq[29][77] = 596; + KRFreq[19][33] = 595; + KRFreq[30][0] = 594; + KRFreq[29][89] = 593; + KRFreq[31][26] = 592; + KRFreq[31][38] = 591; + KRFreq[32][85] = 590; + KRFreq[15][0] = 589; + KRFreq[16][54] = 588; + KRFreq[15][76] = 587; + KRFreq[31][25] = 586; + KRFreq[23][13] = 585; + KRFreq[28][34] = 584; + KRFreq[18][9] = 583; + KRFreq[29][37] = 582; + KRFreq[22][45] = 581; + KRFreq[19][46] = 580; + KRFreq[16][65] = 579; + KRFreq[23][5] = 578; + KRFreq[26][70] = 577; + KRFreq[31][53] = 576; + KRFreq[27][12] = 575; + KRFreq[30][67] = 574; + KRFreq[31][57] = 573; + KRFreq[20][20] = 572; + KRFreq[30][31] = 571; + KRFreq[20][72] = 570; + KRFreq[15][51] = 569; + KRFreq[3][8] = 568; + KRFreq[32][53] = 567; + KRFreq[27][85] = 566; + KRFreq[25][23] = 565; + KRFreq[15][44] = 564; + KRFreq[32][3] = 563; + KRFreq[31][68] = 562; + KRFreq[30][24] = 561; + KRFreq[29][49] = 560; + KRFreq[27][49] = 559; + KRFreq[23][23] = 558; + KRFreq[31][91] = 557; + KRFreq[31][46] = 556; + KRFreq[19][74] = 555; + KRFreq[27][27] = 554; + KRFreq[3][17] = 553; + KRFreq[20][38] = 552; + KRFreq[21][82] = 551; + KRFreq[28][25] = 550; + KRFreq[32][5] = 549; + KRFreq[31][23] = 548; + KRFreq[25][45] = 547; + KRFreq[32][87] = 546; + KRFreq[18][26] = 545; + KRFreq[24][10] = 544; + KRFreq[26][82] = 543; + KRFreq[15][89] = 542; + KRFreq[28][36] = 541; + KRFreq[28][31] = 540; + KRFreq[16][23] = 539; + KRFreq[16][77] = 538; + KRFreq[19][84] = 537; + KRFreq[23][72] = 536; + KRFreq[38][48] = 535; + KRFreq[23][2] = 534; + KRFreq[30][20] = 533; + KRFreq[38][47] = 532; + KRFreq[39][12] = 531; + KRFreq[23][21] = 530; + KRFreq[18][17] = 529; + KRFreq[30][87] = 528; + KRFreq[29][62] = 527; + KRFreq[29][87] = 526; + KRFreq[34][53] = 525; + KRFreq[32][29] = 524; + KRFreq[35][0] = 523; + KRFreq[24][43] = 522; + KRFreq[36][44] = 521; + KRFreq[20][30] = 520; + KRFreq[39][86] = 519; + KRFreq[22][14] = 518; + KRFreq[29][39] = 517; + KRFreq[28][38] = 516; + KRFreq[23][79] = 515; + KRFreq[24][56] = 514; + KRFreq[29][63] = 513; + KRFreq[31][45] = 512; + KRFreq[23][26] = 511; + KRFreq[15][87] = 510; + KRFreq[30][74] = 509; + KRFreq[24][69] = 508; + KRFreq[20][4] = 507; + KRFreq[27][50] = 506; + KRFreq[30][75] = 505; + KRFreq[24][13] = 504; + KRFreq[30][8] = 503; + KRFreq[31][6] = 502; + KRFreq[25][80] = 501; + KRFreq[36][8] = 500; + KRFreq[15][18] = 499; + KRFreq[39][23] = 498; + KRFreq[16][24] = 497; + KRFreq[31][89] = 496; + KRFreq[15][71] = 495; + KRFreq[15][57] = 494; + KRFreq[30][11] = 493; + KRFreq[15][36] = 492; + KRFreq[16][60] = 491; + KRFreq[24][45] = 490; + KRFreq[37][35] = 489; + KRFreq[24][87] = 488; + KRFreq[20][45] = 487; + KRFreq[31][90] = 486; + KRFreq[32][21] = 485; + KRFreq[19][70] = 484; + KRFreq[24][15] = 483; + KRFreq[26][92] = 482; + KRFreq[37][13] = 481; + KRFreq[39][2] = 480; + KRFreq[23][70] = 479; + KRFreq[27][25] = 478; + KRFreq[15][69] = 477; + KRFreq[19][61] = 476; + KRFreq[31][58] = 475; + KRFreq[24][57] = 474; + KRFreq[36][74] = 473; + KRFreq[21][6] = 472; + KRFreq[30][44] = 471; + KRFreq[15][91] = 470; + KRFreq[27][16] = 469; + KRFreq[29][42] = 468; + KRFreq[33][86] = 467; + KRFreq[29][41] = 466; + KRFreq[20][68] = 465; + KRFreq[25][47] = 464; + KRFreq[22][0] = 463; + KRFreq[18][14] = 462; + KRFreq[31][28] = 461; + KRFreq[15][2] = 460; + KRFreq[23][76] = 459; + KRFreq[38][32] = 458; + KRFreq[29][82] = 457; + KRFreq[21][86] = 456; + KRFreq[24][62] = 455; + KRFreq[31][64] = 454; + KRFreq[38][26] = 453; + KRFreq[32][86] = 452; + KRFreq[22][32] = 451; + KRFreq[19][59] = 450; + KRFreq[34][18] = 449; + KRFreq[18][54] = 448; + KRFreq[38][63] = 447; + KRFreq[36][23] = 446; + KRFreq[35][35] = 445; + KRFreq[32][62] = 444; + KRFreq[28][35] = 443; + KRFreq[27][13] = 442; + KRFreq[31][59] = 441; + KRFreq[29][29] = 440; + KRFreq[15][64] = 439; + KRFreq[26][84] = 438; + KRFreq[21][90] = 437; + KRFreq[20][24] = 436; + KRFreq[16][18] = 435; + KRFreq[22][23] = 434; + KRFreq[31][14] = 433; + KRFreq[15][1] = 432; + KRFreq[18][63] = 431; + KRFreq[19][10] = 430; + KRFreq[25][49] = 429; + KRFreq[36][57] = 428; + KRFreq[20][22] = 427; + KRFreq[15][15] = 426; + KRFreq[31][51] = 425; + KRFreq[24][60] = 424; + KRFreq[31][70] = 423; + KRFreq[15][7] = 422; + KRFreq[28][40] = 421; + KRFreq[18][41] = 420; + KRFreq[15][38] = 419; + KRFreq[32][0] = 418; + KRFreq[19][51] = 417; + KRFreq[34][62] = 416; + KRFreq[16][27] = 415; + KRFreq[20][70] = 414; + KRFreq[22][33] = 413; + KRFreq[26][73] = 412; + KRFreq[20][79] = 411; + KRFreq[23][6] = 410; + KRFreq[24][85] = 409; + KRFreq[38][51] = 408; + KRFreq[29][88] = 407; + KRFreq[38][55] = 406; + KRFreq[32][32] = 405; + KRFreq[27][18] = 404; + KRFreq[23][87] = 403; + KRFreq[35][6] = 402; + KRFreq[34][27] = 401; + KRFreq[39][35] = 400; + KRFreq[30][88] = 399; + KRFreq[32][92] = 398; + KRFreq[32][49] = 397; + KRFreq[24][61] = 396; + KRFreq[18][74] = 395; + KRFreq[23][77] = 394; + KRFreq[23][50] = 393; + KRFreq[23][32] = 392; + KRFreq[23][36] = 391; + KRFreq[38][38] = 390; + KRFreq[29][86] = 389; + KRFreq[36][15] = 388; + KRFreq[31][50] = 387; + KRFreq[15][86] = 386; + KRFreq[39][13] = 385; + KRFreq[34][26] = 384; + KRFreq[19][34] = 383; + KRFreq[16][3] = 382; + KRFreq[26][93] = 381; + KRFreq[19][67] = 380; + KRFreq[24][72] = 379; + KRFreq[29][17] = 378; + KRFreq[23][24] = 377; + KRFreq[25][19] = 376; + KRFreq[18][65] = 375; + KRFreq[30][78] = 374; + KRFreq[27][52] = 373; + KRFreq[22][18] = 372; + KRFreq[16][38] = 371; + KRFreq[21][26] = 370; + KRFreq[34][20] = 369; + KRFreq[15][42] = 368; + KRFreq[16][71] = 367; + KRFreq[17][17] = 366; + KRFreq[24][71] = 365; + KRFreq[18][84] = 364; + KRFreq[15][40] = 363; + KRFreq[31][62] = 362; + KRFreq[15][8] = 361; + KRFreq[16][69] = 360; + KRFreq[29][79] = 359; + KRFreq[38][91] = 358; + KRFreq[31][92] = 357; + KRFreq[20][77] = 356; + KRFreq[3][16] = 355; + KRFreq[27][87] = 354; + KRFreq[16][25] = 353; + KRFreq[36][33] = 352; + KRFreq[37][76] = 351; + KRFreq[30][12] = 350; + KRFreq[26][75] = 349; + KRFreq[25][14] = 348; + KRFreq[32][26] = 347; + KRFreq[23][22] = 346; + KRFreq[20][90] = 345; + KRFreq[19][8] = 344; + KRFreq[38][41] = 343; + KRFreq[34][2] = 342; + KRFreq[39][4] = 341; + KRFreq[27][89] = 340; + KRFreq[28][41] = 339; + KRFreq[28][44] = 338; + KRFreq[24][92] = 337; + KRFreq[34][65] = 336; + KRFreq[39][14] = 335; + KRFreq[21][38] = 334; + KRFreq[19][31] = 333; + KRFreq[37][39] = 332; + KRFreq[33][41] = 331; + KRFreq[38][4] = 330; + KRFreq[23][80] = 329; + KRFreq[25][24] = 328; + KRFreq[37][17] = 327; + KRFreq[22][16] = 326; + KRFreq[22][46] = 325; + KRFreq[33][91] = 324; + KRFreq[24][89] = 323; + KRFreq[30][52] = 322; + KRFreq[29][38] = 321; + KRFreq[38][85] = 320; + KRFreq[15][12] = 319; + KRFreq[27][58] = 318; + KRFreq[29][52] = 317; + KRFreq[37][38] = 316; + KRFreq[34][41] = 315; + KRFreq[31][65] = 314; + KRFreq[29][53] = 313; + KRFreq[22][47] = 312; + KRFreq[22][19] = 311; + KRFreq[26][0] = 310; + KRFreq[37][86] = 309; + KRFreq[35][4] = 308; + KRFreq[36][54] = 307; + KRFreq[20][76] = 306; + KRFreq[30][9] = 305; + KRFreq[30][33] = 304; + KRFreq[23][17] = 303; + KRFreq[23][33] = 302; + KRFreq[38][52] = 301; + KRFreq[15][19] = 300; + KRFreq[28][45] = 299; + KRFreq[29][78] = 298; + KRFreq[23][15] = 297; + KRFreq[33][5] = 296; + KRFreq[17][40] = 295; + KRFreq[30][83] = 294; + KRFreq[18][1] = 293; + KRFreq[30][81] = 292; + KRFreq[19][40] = 291; + KRFreq[24][47] = 290; + KRFreq[17][56] = 289; + KRFreq[39][80] = 288; + KRFreq[30][46] = 287; + KRFreq[16][61] = 286; + KRFreq[26][78] = 285; + KRFreq[26][57] = 284; + KRFreq[20][46] = 283; + KRFreq[25][15] = 282; + KRFreq[25][91] = 281; + KRFreq[21][83] = 280; + KRFreq[30][77] = 279; + KRFreq[35][30] = 278; + KRFreq[30][34] = 277; + KRFreq[20][69] = 276; + KRFreq[35][10] = 275; + KRFreq[29][70] = 274; + KRFreq[22][50] = 273; + KRFreq[18][0] = 272; + KRFreq[22][64] = 271; + KRFreq[38][65] = 270; + KRFreq[22][70] = 269; + KRFreq[24][58] = 268; + KRFreq[19][66] = 267; + KRFreq[30][59] = 266; + KRFreq[37][14] = 265; + KRFreq[16][56] = 264; + KRFreq[29][85] = 263; + KRFreq[31][15] = 262; + KRFreq[36][84] = 261; + KRFreq[39][15] = 260; + KRFreq[39][90] = 259; + KRFreq[18][12] = 258; + KRFreq[21][93] = 257; + KRFreq[24][66] = 256; + KRFreq[27][90] = 255; + KRFreq[25][90] = 254; + KRFreq[22][24] = 253; + KRFreq[36][67] = 252; + KRFreq[33][90] = 251; + KRFreq[15][60] = 250; + KRFreq[23][85] = 249; + KRFreq[34][1] = 248; + KRFreq[39][37] = 247; + KRFreq[21][18] = 246; + KRFreq[34][4] = 245; + KRFreq[28][33] = 244; + KRFreq[15][13] = 243; + KRFreq[32][22] = 242; + KRFreq[30][76] = 241; + KRFreq[20][21] = 240; + KRFreq[38][66] = 239; + KRFreq[32][55] = 238; + KRFreq[32][89] = 237; + KRFreq[25][26] = 236; + KRFreq[16][80] = 235; + KRFreq[15][43] = 234; + KRFreq[38][54] = 233; + KRFreq[39][68] = 232; + KRFreq[22][88] = 231; + KRFreq[21][84] = 230; + KRFreq[21][17] = 229; + KRFreq[20][28] = 228; + KRFreq[32][1] = 227; + KRFreq[33][87] = 226; + KRFreq[38][71] = 225; + KRFreq[37][47] = 224; + KRFreq[18][77] = 223; + KRFreq[37][58] = 222; + KRFreq[34][74] = 221; + KRFreq[32][54] = 220; + KRFreq[27][33] = 219; + KRFreq[32][93] = 218; + KRFreq[23][51] = 217; + KRFreq[20][57] = 216; + KRFreq[22][37] = 215; + KRFreq[39][10] = 214; + KRFreq[39][17] = 213; + KRFreq[33][4] = 212; + KRFreq[32][84] = 211; + KRFreq[34][3] = 210; + KRFreq[28][27] = 209; + KRFreq[15][79] = 208; + KRFreq[34][21] = 207; + KRFreq[34][69] = 206; + KRFreq[21][62] = 205; + KRFreq[36][24] = 204; + KRFreq[16][89] = 203; + KRFreq[18][48] = 202; + KRFreq[38][15] = 201; + KRFreq[36][58] = 200; + KRFreq[21][56] = 199; + KRFreq[34][48] = 198; + KRFreq[21][15] = 197; + KRFreq[39][3] = 196; + KRFreq[16][44] = 195; + KRFreq[18][79] = 194; + KRFreq[25][13] = 193; + KRFreq[29][47] = 192; + KRFreq[38][88] = 191; + KRFreq[20][71] = 190; + KRFreq[16][58] = 189; + KRFreq[35][57] = 188; + KRFreq[29][30] = 187; + KRFreq[29][23] = 186; + KRFreq[34][93] = 185; + KRFreq[30][85] = 184; + KRFreq[15][80] = 183; + KRFreq[32][78] = 182; + KRFreq[37][82] = 181; + KRFreq[22][40] = 180; + KRFreq[21][69] = 179; + KRFreq[26][85] = 178; + KRFreq[31][31] = 177; + KRFreq[28][64] = 176; + KRFreq[38][13] = 175; + KRFreq[25][2] = 174; + KRFreq[22][34] = 173; + KRFreq[28][28] = 172; + KRFreq[24][91] = 171; + KRFreq[33][74] = 170; + KRFreq[29][40] = 169; + KRFreq[15][77] = 168; + KRFreq[32][80] = 167; + KRFreq[30][41] = 166; + KRFreq[23][30] = 165; + KRFreq[24][63] = 164; + KRFreq[30][53] = 163; + KRFreq[39][70] = 162; + KRFreq[23][61] = 161; + KRFreq[37][27] = 160; + KRFreq[16][55] = 159; + KRFreq[22][74] = 158; + KRFreq[26][50] = 157; + KRFreq[16][10] = 156; + KRFreq[34][63] = 155; + KRFreq[35][14] = 154; + KRFreq[17][7] = 153; + KRFreq[15][59] = 152; + KRFreq[27][23] = 151; + KRFreq[18][70] = 150; + KRFreq[32][56] = 149; + KRFreq[37][87] = 148; + KRFreq[17][61] = 147; + KRFreq[18][83] = 146; + KRFreq[23][86] = 145; + KRFreq[17][31] = 144; + KRFreq[23][83] = 143; + KRFreq[35][2] = 142; + KRFreq[18][64] = 141; + KRFreq[27][43] = 140; + KRFreq[32][42] = 139; + KRFreq[25][76] = 138; + KRFreq[19][85] = 137; + KRFreq[37][81] = 136; + KRFreq[38][83] = 135; + KRFreq[35][7] = 134; + KRFreq[16][51] = 133; + KRFreq[27][22] = 132; + KRFreq[16][76] = 131; + KRFreq[22][4] = 130; + KRFreq[38][84] = 129; + KRFreq[17][83] = 128; + KRFreq[24][46] = 127; + KRFreq[33][15] = 126; + KRFreq[20][48] = 125; + KRFreq[17][30] = 124; + KRFreq[30][93] = 123; + KRFreq[28][11] = 122; + KRFreq[28][30] = 121; + KRFreq[15][62] = 120; + KRFreq[17][87] = 119; + KRFreq[32][81] = 118; + KRFreq[23][37] = 117; + KRFreq[30][22] = 116; + KRFreq[32][66] = 115; + KRFreq[33][78] = 114; + KRFreq[21][4] = 113; + KRFreq[31][17] = 112; + KRFreq[39][61] = 111; + KRFreq[18][76] = 110; + KRFreq[15][85] = 109; + KRFreq[31][47] = 108; + KRFreq[19][57] = 107; + KRFreq[23][55] = 106; + KRFreq[27][29] = 105; + KRFreq[29][46] = 104; + KRFreq[33][0] = 103; + KRFreq[16][83] = 102; + KRFreq[39][78] = 101; + KRFreq[32][77] = 100; + KRFreq[36][25] = 99; + KRFreq[34][19] = 98; + KRFreq[38][49] = 97; + KRFreq[19][25] = 96; + KRFreq[23][53] = 95; + KRFreq[28][43] = 94; + KRFreq[31][44] = 93; + KRFreq[36][34] = 92; + KRFreq[16][34] = 91; + KRFreq[35][1] = 90; + KRFreq[19][87] = 89; + KRFreq[18][53] = 88; + KRFreq[29][54] = 87; + KRFreq[22][41] = 86; + KRFreq[38][18] = 85; + KRFreq[22][2] = 84; + KRFreq[20][3] = 83; + KRFreq[39][69] = 82; + KRFreq[30][29] = 81; + KRFreq[28][19] = 80; + KRFreq[29][90] = 79; + KRFreq[17][86] = 78; + KRFreq[15][9] = 77; + KRFreq[39][73] = 76; + KRFreq[15][37] = 75; + KRFreq[35][40] = 74; + KRFreq[33][77] = 73; + KRFreq[27][86] = 72; + KRFreq[36][79] = 71; + KRFreq[23][18] = 70; + KRFreq[34][87] = 69; + KRFreq[39][24] = 68; + KRFreq[26][8] = 67; + KRFreq[33][48] = 66; + KRFreq[39][30] = 65; + KRFreq[33][28] = 64; + KRFreq[16][67] = 63; + KRFreq[31][78] = 62; + KRFreq[32][23] = 61; + KRFreq[24][55] = 60; + KRFreq[30][68] = 59; + KRFreq[18][60] = 58; + KRFreq[15][17] = 57; + KRFreq[23][34] = 56; + KRFreq[20][49] = 55; + KRFreq[15][78] = 54; + KRFreq[24][14] = 53; + KRFreq[19][41] = 52; + KRFreq[31][55] = 51; + KRFreq[21][39] = 50; + KRFreq[35][9] = 49; + KRFreq[30][15] = 48; + KRFreq[20][52] = 47; + KRFreq[35][71] = 46; + KRFreq[20][7] = 45; + KRFreq[29][72] = 44; + KRFreq[37][77] = 43; + KRFreq[22][35] = 42; + KRFreq[20][61] = 41; + KRFreq[31][60] = 40; + KRFreq[20][93] = 39; + KRFreq[27][92] = 38; + KRFreq[28][16] = 37; + KRFreq[36][26] = 36; + KRFreq[18][89] = 35; + KRFreq[21][63] = 34; + KRFreq[22][52] = 33; + KRFreq[24][65] = 32; + KRFreq[31][8] = 31; + KRFreq[31][49] = 30; + KRFreq[33][30] = 29; + KRFreq[37][15] = 28; + KRFreq[18][18] = 27; + KRFreq[25][50] = 26; + KRFreq[29][20] = 25; + KRFreq[35][48] = 24; + KRFreq[38][75] = 23; + KRFreq[26][83] = 22; + KRFreq[21][87] = 21; + KRFreq[27][71] = 20; + KRFreq[32][91] = 19; + KRFreq[25][73] = 18; + KRFreq[16][84] = 17; + KRFreq[25][31] = 16; + KRFreq[17][90] = 15; + KRFreq[18][40] = 14; + KRFreq[17][77] = 13; + KRFreq[17][35] = 12; + KRFreq[23][52] = 11; + KRFreq[23][35] = 10; + KRFreq[16][5] = 9; + KRFreq[23][58] = 8; + KRFreq[19][60] = 7; + KRFreq[30][32] = 6; + KRFreq[38][34] = 5; + KRFreq[23][4] = 4; + KRFreq[23][1] = 3; + KRFreq[27][57] = 2; + KRFreq[39][38] = 1; + KRFreq[32][33] = 0; + JPFreq[3][74] = 600; + JPFreq[3][45] = 599; + JPFreq[3][3] = 598; + JPFreq[3][24] = 597; + JPFreq[3][30] = 596; + JPFreq[3][42] = 595; + JPFreq[3][46] = 594; + JPFreq[3][39] = 593; + JPFreq[3][11] = 592; + JPFreq[3][37] = 591; + JPFreq[3][38] = 590; + JPFreq[3][31] = 589; + JPFreq[3][41] = 588; + JPFreq[3][5] = 587; + JPFreq[3][10] = 586; + JPFreq[3][75] = 585; + JPFreq[3][65] = 584; + JPFreq[3][72] = 583; + JPFreq[37][91] = 582; + JPFreq[0][27] = 581; + JPFreq[3][18] = 580; + JPFreq[3][22] = 579; + JPFreq[3][61] = 578; + JPFreq[3][14] = 577; + JPFreq[24][80] = 576; + JPFreq[4][82] = 575; + JPFreq[17][80] = 574; + JPFreq[30][44] = 573; + JPFreq[3][73] = 572; + JPFreq[3][64] = 571; + JPFreq[38][14] = 570; + JPFreq[33][70] = 569; + JPFreq[3][1] = 568; + JPFreq[3][16] = 567; + JPFreq[3][35] = 566; + JPFreq[3][40] = 565; + JPFreq[4][74] = 564; + JPFreq[4][24] = 563; + JPFreq[42][59] = 562; + JPFreq[3][7] = 561; + JPFreq[3][71] = 560; + JPFreq[3][12] = 559; + JPFreq[15][75] = 558; + JPFreq[3][20] = 557; + JPFreq[4][39] = 556; + JPFreq[34][69] = 555; + JPFreq[3][28] = 554; + JPFreq[35][24] = 553; + JPFreq[3][82] = 552; + JPFreq[28][47] = 551; + JPFreq[3][67] = 550; + JPFreq[37][16] = 549; + JPFreq[26][93] = 548; + JPFreq[4][1] = 547; + JPFreq[26][85] = 546; + JPFreq[31][14] = 545; + JPFreq[4][3] = 544; + JPFreq[4][72] = 543; + JPFreq[24][51] = 542; + JPFreq[27][51] = 541; + JPFreq[27][49] = 540; + JPFreq[22][77] = 539; + JPFreq[27][10] = 538; + JPFreq[29][68] = 537; + JPFreq[20][35] = 536; + JPFreq[41][11] = 535; + JPFreq[24][70] = 534; + JPFreq[36][61] = 533; + JPFreq[31][23] = 532; + JPFreq[43][16] = 531; + JPFreq[23][68] = 530; + JPFreq[32][15] = 529; + JPFreq[3][32] = 528; + JPFreq[19][53] = 527; + JPFreq[40][83] = 526; + JPFreq[4][14] = 525; + JPFreq[36][9] = 524; + JPFreq[4][73] = 523; + JPFreq[23][10] = 522; + JPFreq[3][63] = 521; + JPFreq[39][14] = 520; + JPFreq[3][78] = 519; + JPFreq[33][47] = 518; + JPFreq[21][39] = 517; + JPFreq[34][46] = 516; + JPFreq[36][75] = 515; + JPFreq[41][92] = 514; + JPFreq[37][93] = 513; + JPFreq[4][34] = 512; + JPFreq[15][86] = 511; + JPFreq[46][1] = 510; + JPFreq[37][65] = 509; + JPFreq[3][62] = 508; + JPFreq[32][73] = 507; + JPFreq[21][65] = 506; + JPFreq[29][75] = 505; + JPFreq[26][51] = 504; + JPFreq[3][34] = 503; + JPFreq[4][10] = 502; + JPFreq[30][22] = 501; + JPFreq[35][73] = 500; + JPFreq[17][82] = 499; + JPFreq[45][8] = 498; + JPFreq[27][73] = 497; + JPFreq[18][55] = 496; + JPFreq[25][2] = 495; + JPFreq[3][26] = 494; + JPFreq[45][46] = 493; + JPFreq[4][22] = 492; + JPFreq[4][40] = 491; + JPFreq[18][10] = 490; + JPFreq[32][9] = 489; + JPFreq[26][49] = 488; + JPFreq[3][47] = 487; + JPFreq[24][65] = 486; + JPFreq[4][76] = 485; + JPFreq[43][67] = 484; + JPFreq[3][9] = 483; + JPFreq[41][37] = 482; + JPFreq[33][68] = 481; + JPFreq[43][31] = 480; + JPFreq[19][55] = 479; + JPFreq[4][30] = 478; + JPFreq[27][33] = 477; + JPFreq[16][62] = 476; + JPFreq[36][35] = 475; + JPFreq[37][15] = 474; + JPFreq[27][70] = 473; + JPFreq[22][71] = 472; + JPFreq[33][45] = 471; + JPFreq[31][78] = 470; + JPFreq[43][59] = 469; + JPFreq[32][19] = 468; + JPFreq[17][28] = 467; + JPFreq[40][28] = 466; + JPFreq[20][93] = 465; + JPFreq[18][15] = 464; + JPFreq[4][23] = 463; + JPFreq[3][23] = 462; + JPFreq[26][64] = 461; + JPFreq[44][92] = 460; + JPFreq[17][27] = 459; + JPFreq[3][56] = 458; + JPFreq[25][38] = 457; + JPFreq[23][31] = 456; + JPFreq[35][43] = 455; + JPFreq[4][54] = 454; + JPFreq[35][19] = 453; + JPFreq[22][47] = 452; + JPFreq[42][0] = 451; + JPFreq[23][28] = 450; + JPFreq[46][33] = 449; + JPFreq[36][85] = 448; + JPFreq[31][12] = 447; + JPFreq[3][76] = 446; + JPFreq[4][75] = 445; + JPFreq[36][56] = 444; + JPFreq[4][64] = 443; + JPFreq[25][77] = 442; + JPFreq[15][52] = 441; + JPFreq[33][73] = 440; + JPFreq[3][55] = 439; + JPFreq[43][82] = 438; + JPFreq[27][82] = 437; + JPFreq[20][3] = 436; + JPFreq[40][51] = 435; + JPFreq[3][17] = 434; + JPFreq[27][71] = 433; + JPFreq[4][52] = 432; + JPFreq[44][48] = 431; + JPFreq[27][2] = 430; + JPFreq[17][39] = 429; + JPFreq[31][8] = 428; + JPFreq[44][54] = 427; + JPFreq[43][18] = 426; + JPFreq[43][77] = 425; + JPFreq[4][61] = 424; + JPFreq[19][91] = 423; + JPFreq[31][13] = 422; + JPFreq[44][71] = 421; + JPFreq[20][0] = 420; + JPFreq[23][87] = 419; + JPFreq[21][14] = 418; + JPFreq[29][13] = 417; + JPFreq[3][58] = 416; + JPFreq[26][18] = 415; + JPFreq[4][47] = 414; + JPFreq[4][18] = 413; + JPFreq[3][53] = 412; + JPFreq[26][92] = 411; + JPFreq[21][7] = 410; + JPFreq[4][37] = 409; + JPFreq[4][63] = 408; + JPFreq[36][51] = 407; + JPFreq[4][32] = 406; + JPFreq[28][73] = 405; + JPFreq[4][50] = 404; + JPFreq[41][60] = 403; + JPFreq[23][1] = 402; + JPFreq[36][92] = 401; + JPFreq[15][41] = 400; + JPFreq[21][71] = 399; + JPFreq[41][30] = 398; + JPFreq[32][76] = 397; + JPFreq[17][34] = 396; + JPFreq[26][15] = 395; + JPFreq[26][25] = 394; + JPFreq[31][77] = 393; + JPFreq[31][3] = 392; + JPFreq[46][34] = 391; + JPFreq[27][84] = 390; + JPFreq[23][8] = 389; + JPFreq[16][0] = 388; + JPFreq[28][80] = 387; + JPFreq[26][54] = 386; + JPFreq[33][18] = 385; + JPFreq[31][20] = 384; + JPFreq[31][62] = 383; + JPFreq[30][41] = 382; + JPFreq[33][30] = 381; + JPFreq[45][45] = 380; + JPFreq[37][82] = 379; + JPFreq[15][33] = 378; + JPFreq[20][12] = 377; + JPFreq[18][5] = 376; + JPFreq[28][86] = 375; + JPFreq[30][19] = 374; + JPFreq[42][43] = 373; + JPFreq[36][31] = 372; + JPFreq[17][93] = 371; + JPFreq[4][15] = 370; + JPFreq[21][20] = 369; + JPFreq[23][21] = 368; + JPFreq[28][72] = 367; + JPFreq[4][20] = 366; + JPFreq[26][55] = 365; + JPFreq[21][5] = 364; + JPFreq[19][16] = 363; + JPFreq[23][64] = 362; + JPFreq[40][59] = 361; + JPFreq[37][26] = 360; + JPFreq[26][56] = 359; + JPFreq[4][12] = 358; + JPFreq[33][71] = 357; + JPFreq[32][39] = 356; + JPFreq[38][40] = 355; + JPFreq[22][74] = 354; + JPFreq[3][25] = 353; + JPFreq[15][48] = 352; + JPFreq[41][82] = 351; + JPFreq[41][9] = 350; + JPFreq[25][48] = 349; + JPFreq[31][71] = 348; + JPFreq[43][29] = 347; + JPFreq[26][80] = 346; + JPFreq[4][5] = 345; + JPFreq[18][71] = 344; + JPFreq[29][0] = 343; + JPFreq[43][43] = 342; + JPFreq[23][81] = 341; + JPFreq[4][42] = 340; + JPFreq[44][28] = 339; + JPFreq[23][93] = 338; + JPFreq[17][81] = 337; + JPFreq[25][25] = 336; + JPFreq[41][23] = 335; + JPFreq[34][35] = 334; + JPFreq[4][53] = 333; + JPFreq[28][36] = 332; + JPFreq[4][41] = 331; + JPFreq[25][60] = 330; + JPFreq[23][20] = 329; + JPFreq[3][43] = 328; + JPFreq[24][79] = 327; + JPFreq[29][41] = 326; + JPFreq[30][83] = 325; + JPFreq[3][50] = 324; + JPFreq[22][18] = 323; + JPFreq[18][3] = 322; + JPFreq[39][30] = 321; + JPFreq[4][28] = 320; + JPFreq[21][64] = 319; + JPFreq[4][68] = 318; + JPFreq[17][71] = 317; + JPFreq[27][0] = 316; + JPFreq[39][28] = 315; + JPFreq[30][13] = 314; + JPFreq[36][70] = 313; + JPFreq[20][82] = 312; + JPFreq[33][38] = 311; + JPFreq[44][87] = 310; + JPFreq[34][45] = 309; + JPFreq[4][26] = 308; + JPFreq[24][44] = 307; + JPFreq[38][67] = 306; + JPFreq[38][6] = 305; + JPFreq[30][68] = 304; + JPFreq[15][89] = 303; + JPFreq[24][93] = 302; + JPFreq[40][41] = 301; + JPFreq[38][3] = 300; + JPFreq[28][23] = 299; + JPFreq[26][17] = 298; + JPFreq[4][38] = 297; + JPFreq[22][78] = 296; + JPFreq[15][37] = 295; + JPFreq[25][85] = 294; + JPFreq[4][9] = 293; + JPFreq[4][7] = 292; + JPFreq[27][53] = 291; + JPFreq[39][29] = 290; + JPFreq[41][43] = 289; + JPFreq[25][62] = 288; + JPFreq[4][48] = 287; + JPFreq[28][28] = 286; + JPFreq[21][40] = 285; + JPFreq[36][73] = 284; + JPFreq[26][39] = 283; + JPFreq[22][54] = 282; + JPFreq[33][5] = 281; + JPFreq[19][21] = 280; + JPFreq[46][31] = 279; + JPFreq[20][64] = 278; + JPFreq[26][63] = 277; + JPFreq[22][23] = 276; + JPFreq[25][81] = 275; + JPFreq[4][62] = 274; + JPFreq[37][31] = 273; + JPFreq[40][52] = 272; + JPFreq[29][79] = 271; + JPFreq[41][48] = 270; + JPFreq[31][57] = 269; + JPFreq[32][92] = 268; + JPFreq[36][36] = 267; + JPFreq[27][7] = 266; + JPFreq[35][29] = 265; + JPFreq[37][34] = 264; + JPFreq[34][42] = 263; + JPFreq[27][15] = 262; + JPFreq[33][27] = 261; + JPFreq[31][38] = 260; + JPFreq[19][79] = 259; + JPFreq[4][31] = 258; + JPFreq[4][66] = 257; + JPFreq[17][32] = 256; + JPFreq[26][67] = 255; + JPFreq[16][30] = 254; + JPFreq[26][46] = 253; + JPFreq[24][26] = 252; + JPFreq[35][10] = 251; + JPFreq[18][37] = 250; + JPFreq[3][19] = 249; + JPFreq[33][69] = 248; + JPFreq[31][9] = 247; + JPFreq[45][29] = 246; + JPFreq[3][15] = 245; + JPFreq[18][54] = 244; + JPFreq[3][44] = 243; + JPFreq[31][29] = 242; + JPFreq[18][45] = 241; + JPFreq[38][28] = 240; + JPFreq[24][12] = 239; + JPFreq[35][82] = 238; + JPFreq[17][43] = 237; + JPFreq[28][9] = 236; + JPFreq[23][25] = 235; + JPFreq[44][37] = 234; + JPFreq[23][75] = 233; + JPFreq[23][92] = 232; + JPFreq[0][24] = 231; + JPFreq[19][74] = 230; + JPFreq[45][32] = 229; + JPFreq[16][72] = 228; + JPFreq[16][93] = 227; + JPFreq[45][13] = 226; + JPFreq[24][8] = 225; + JPFreq[25][47] = 224; + JPFreq[28][26] = 223; + JPFreq[43][81] = 222; + JPFreq[32][71] = 221; + JPFreq[18][41] = 220; + JPFreq[26][62] = 219; + JPFreq[41][24] = 218; + JPFreq[40][11] = 217; + JPFreq[43][57] = 216; + JPFreq[34][53] = 215; + JPFreq[20][32] = 214; + JPFreq[34][43] = 213; + JPFreq[41][91] = 212; + JPFreq[29][57] = 211; + JPFreq[15][43] = 210; + JPFreq[22][89] = 209; + JPFreq[33][83] = 208; + JPFreq[43][20] = 207; + JPFreq[25][58] = 206; + JPFreq[30][30] = 205; + JPFreq[4][56] = 204; + JPFreq[17][64] = 203; + JPFreq[23][0] = 202; + JPFreq[44][12] = 201; + JPFreq[25][37] = 200; + JPFreq[35][13] = 199; + JPFreq[20][30] = 198; + JPFreq[21][84] = 197; + JPFreq[29][14] = 196; + JPFreq[30][5] = 195; + JPFreq[37][2] = 194; + JPFreq[4][78] = 193; + JPFreq[29][78] = 192; + JPFreq[29][84] = 191; + JPFreq[32][86] = 190; + JPFreq[20][68] = 189; + JPFreq[30][39] = 188; + JPFreq[15][69] = 187; + JPFreq[4][60] = 186; + JPFreq[20][61] = 185; + JPFreq[41][67] = 184; + JPFreq[16][35] = 183; + JPFreq[36][57] = 182; + JPFreq[39][80] = 181; + JPFreq[4][59] = 180; + JPFreq[4][44] = 179; + JPFreq[40][54] = 178; + JPFreq[30][8] = 177; + JPFreq[44][30] = 176; + JPFreq[31][93] = 175; + JPFreq[31][47] = 174; + JPFreq[16][70] = 173; + JPFreq[21][0] = 172; + JPFreq[17][35] = 171; + JPFreq[21][67] = 170; + JPFreq[44][18] = 169; + JPFreq[36][29] = 168; + JPFreq[18][67] = 167; + JPFreq[24][28] = 166; + JPFreq[36][24] = 165; + JPFreq[23][5] = 164; + JPFreq[31][65] = 163; + JPFreq[26][59] = 162; + JPFreq[28][2] = 161; + JPFreq[39][69] = 160; + JPFreq[42][40] = 159; + JPFreq[37][80] = 158; + JPFreq[15][66] = 157; + JPFreq[34][38] = 156; + JPFreq[28][48] = 155; + JPFreq[37][77] = 154; + JPFreq[29][34] = 153; + JPFreq[33][12] = 152; + JPFreq[4][65] = 151; + JPFreq[30][31] = 150; + JPFreq[27][92] = 149; + JPFreq[4][2] = 148; + JPFreq[4][51] = 147; + JPFreq[23][77] = 146; + JPFreq[4][35] = 145; + JPFreq[3][13] = 144; + JPFreq[26][26] = 143; + JPFreq[44][4] = 142; + JPFreq[39][53] = 141; + JPFreq[20][11] = 140; + JPFreq[40][33] = 139; + JPFreq[45][7] = 138; + JPFreq[4][70] = 137; + JPFreq[3][49] = 136; + JPFreq[20][59] = 135; + JPFreq[21][12] = 134; + JPFreq[33][53] = 133; + JPFreq[20][14] = 132; + JPFreq[37][18] = 131; + JPFreq[18][17] = 130; + JPFreq[36][23] = 129; + JPFreq[18][57] = 128; + JPFreq[26][74] = 127; + JPFreq[35][2] = 126; + JPFreq[38][58] = 125; + JPFreq[34][68] = 124; + JPFreq[29][81] = 123; + JPFreq[20][69] = 122; + JPFreq[39][86] = 121; + JPFreq[4][16] = 120; + JPFreq[16][49] = 119; + JPFreq[15][72] = 118; + JPFreq[26][35] = 117; + JPFreq[32][14] = 116; + JPFreq[40][90] = 115; + JPFreq[33][79] = 114; + JPFreq[35][4] = 113; + JPFreq[23][33] = 112; + JPFreq[19][19] = 111; + JPFreq[31][41] = 110; + JPFreq[44][1] = 109; + JPFreq[22][56] = 108; + JPFreq[31][27] = 107; + JPFreq[32][18] = 106; + JPFreq[27][32] = 105; + JPFreq[37][39] = 104; + JPFreq[42][11] = 103; + JPFreq[29][71] = 102; + JPFreq[32][58] = 101; + JPFreq[46][10] = 100; + JPFreq[17][30] = 99; + JPFreq[38][15] = 98; + JPFreq[29][60] = 97; + JPFreq[4][11] = 96; + JPFreq[38][31] = 95; + JPFreq[40][79] = 94; + JPFreq[28][49] = 93; + JPFreq[28][84] = 92; + JPFreq[26][77] = 91; + JPFreq[22][32] = 90; + JPFreq[33][17] = 89; + JPFreq[23][18] = 88; + JPFreq[32][64] = 87; + JPFreq[4][6] = 86; + JPFreq[33][51] = 85; + JPFreq[44][77] = 84; + JPFreq[29][5] = 83; + JPFreq[46][25] = 82; + JPFreq[19][58] = 81; + JPFreq[4][46] = 80; + JPFreq[15][71] = 79; + JPFreq[18][58] = 78; + JPFreq[26][45] = 77; + JPFreq[45][66] = 76; + JPFreq[34][10] = 75; + JPFreq[19][37] = 74; + JPFreq[33][65] = 73; + JPFreq[44][52] = 72; + JPFreq[16][38] = 71; + JPFreq[36][46] = 70; + JPFreq[20][26] = 69; + JPFreq[30][37] = 68; + JPFreq[4][58] = 67; + JPFreq[43][2] = 66; + JPFreq[30][18] = 65; + JPFreq[19][35] = 64; + JPFreq[15][68] = 63; + JPFreq[3][36] = 62; + JPFreq[35][40] = 61; + JPFreq[36][32] = 60; + JPFreq[37][14] = 59; + JPFreq[17][11] = 58; + JPFreq[19][78] = 57; + JPFreq[37][11] = 56; + JPFreq[28][63] = 55; + JPFreq[29][61] = 54; + JPFreq[33][3] = 53; + JPFreq[41][52] = 52; + JPFreq[33][63] = 51; + JPFreq[22][41] = 50; + JPFreq[4][19] = 49; + JPFreq[32][41] = 48; + JPFreq[24][4] = 47; + JPFreq[31][28] = 46; + JPFreq[43][30] = 45; + JPFreq[17][3] = 44; + JPFreq[43][70] = 43; + JPFreq[34][19] = 42; + JPFreq[20][77] = 41; + JPFreq[18][83] = 40; + JPFreq[17][15] = 39; + JPFreq[23][61] = 38; + JPFreq[40][27] = 37; + JPFreq[16][48] = 36; + JPFreq[39][78] = 35; + JPFreq[41][53] = 34; + JPFreq[40][91] = 33; + JPFreq[40][72] = 32; + JPFreq[18][52] = 31; + JPFreq[35][66] = 30; + JPFreq[39][93] = 29; + JPFreq[19][48] = 28; + JPFreq[26][36] = 27; + JPFreq[27][25] = 26; + JPFreq[42][71] = 25; + JPFreq[42][85] = 24; + JPFreq[26][48] = 23; + JPFreq[28][15] = 22; + JPFreq[3][66] = 21; + JPFreq[25][24] = 20; + JPFreq[27][43] = 19; + JPFreq[27][78] = 18; + JPFreq[45][43] = 17; + JPFreq[27][72] = 16; + JPFreq[40][29] = 15; + JPFreq[41][0] = 14; + JPFreq[19][57] = 13; + JPFreq[15][59] = 12; + JPFreq[29][29] = 11; + JPFreq[4][25] = 10; + JPFreq[21][42] = 9; + JPFreq[23][35] = 8; + JPFreq[33][1] = 7; + JPFreq[4][57] = 6; + JPFreq[17][60] = 5; + JPFreq[25][19] = 4; + JPFreq[22][65] = 3; + JPFreq[42][29] = 2; + JPFreq[27][66] = 1; + JPFreq[26][89] = 0; + } +} + +class Encoding { + // Supported Encoding Types + public static int GB2312 = 0; + + public static int GBK = 1; + + public static int GB18030 = 2; + + public static int HZ = 3; + + public static int BIG5 = 4; + + public static int CNS11643 = 5; + + public static int UTF8 = 6; + + public static int UTF8T = 7; + + public static int UTF8S = 8; + + public static int UNICODE = 9; + + public static int UNICODET = 10; + + public static int UNICODES = 11; + + public static int ISO2022CN = 12; + + public static int ISO2022CN_CNS = 13; + + public static int ISO2022CN_GB = 14; + + public static int EUC_KR = 15; + + public static int CP949 = 16; + + public static int ISO2022KR = 17; + + public static int JOHAB = 18; + + public static int SJIS = 19; + + public static int EUC_JP = 20; + + public static int ISO2022JP = 21; + + public static int ASCII = 22; + + public static int OTHER = 23; + + public static int TOTALTYPES = 24; + + public final static int SIMP = 0; + + public final static int TRAD = 1; + + // Names of the encodings as understood by Java + public static String[] javaname; + + // Names of the encodings for human viewing + public static String[] nicename; + + // Names of charsets as used in charset parameter of HTML Meta tag + public static String[] htmlname; + + // Constructor + public Encoding() { + javaname = new String[TOTALTYPES]; + nicename = new String[TOTALTYPES]; + htmlname = new String[TOTALTYPES]; + // Assign encoding names + javaname[GB2312] = "GB2312"; + javaname[GBK] = "GBK"; + javaname[GB18030] = "GB18030"; + javaname[HZ] = "ASCII"; // What to put here? Sun doesn't support HZ + javaname[ISO2022CN_GB] = "ISO2022CN_GB"; + javaname[BIG5] = "BIG5"; + javaname[CNS11643] = "EUC-TW"; + javaname[ISO2022CN_CNS] = "ISO2022CN_CNS"; + javaname[ISO2022CN] = "ISO2022CN"; + javaname[UTF8] = "UTF-8"; + javaname[UTF8T] = "UTF-8"; + javaname[UTF8S] = "UTF-8"; + javaname[UNICODE] = "Unicode"; + javaname[UNICODET] = "Unicode"; + javaname[UNICODES] = "Unicode"; + javaname[EUC_KR] = "EUC_KR"; + javaname[CP949] = "MS949"; + javaname[ISO2022KR] = "ISO2022KR"; + javaname[JOHAB] = "Johab"; + javaname[SJIS] = "SJIS"; + javaname[EUC_JP] = "EUC_JP"; + javaname[ISO2022JP] = "ISO2022JP"; + javaname[ASCII] = "ASCII"; + javaname[OTHER] = "ISO8859_1"; + // Assign encoding names + htmlname[GB2312] = "GB2312"; + htmlname[GBK] = "GBK"; + htmlname[GB18030] = "GB18030"; + htmlname[HZ] = "HZ-GB-2312"; + htmlname[ISO2022CN_GB] = "ISO-2022-CN-EXT"; + htmlname[BIG5] = "BIG5"; + htmlname[CNS11643] = "EUC-TW"; + htmlname[ISO2022CN_CNS] = "ISO-2022-CN-EXT"; + htmlname[ISO2022CN] = "ISO-2022-CN"; + htmlname[UTF8] = "UTF-8"; + htmlname[UTF8T] = "UTF-8"; + htmlname[UTF8S] = "UTF-8"; + htmlname[UNICODE] = "UTF-16"; + htmlname[UNICODET] = "UTF-16"; + htmlname[UNICODES] = "UTF-16"; + htmlname[EUC_KR] = "EUC-KR"; + htmlname[CP949] = "x-windows-949"; + htmlname[ISO2022KR] = "ISO-2022-KR"; + htmlname[JOHAB] = "x-Johab"; + htmlname[SJIS] = "Shift_JIS"; + htmlname[EUC_JP] = "EUC-JP"; + htmlname[ISO2022JP] = "ISO-2022-JP"; + htmlname[ASCII] = "ASCII"; + htmlname[OTHER] = "ISO8859-1"; + // Assign Human readable names + nicename[GB2312] = "GB-2312"; + nicename[GBK] = "GBK"; + nicename[GB18030] = "GB18030"; + nicename[HZ] = "HZ"; + nicename[ISO2022CN_GB] = "ISO2022CN-GB"; + nicename[BIG5] = "Big5"; + nicename[CNS11643] = "CNS11643"; + nicename[ISO2022CN_CNS] = "ISO2022CN-CNS"; + nicename[ISO2022CN] = "ISO2022 CN"; + nicename[UTF8] = "UTF-8"; + nicename[UTF8T] = "UTF-8 (Trad)"; + nicename[UTF8S] = "UTF-8 (Simp)"; + nicename[UNICODE] = "Unicode"; + nicename[UNICODET] = "Unicode (Trad)"; + nicename[UNICODES] = "Unicode (Simp)"; + nicename[EUC_KR] = "EUC-KR"; + nicename[CP949] = "CP949"; + nicename[ISO2022KR] = "ISO 2022 KR"; + nicename[JOHAB] = "Johab"; + nicename[SJIS] = "Shift-JIS"; + nicename[EUC_JP] = "EUC-JP"; + nicename[ISO2022JP] = "ISO 2022 JP"; + nicename[ASCII] = "ASCII"; + nicename[OTHER] = "OTHER"; + } + +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewArtifactInfoResolver.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewArtifactInfoResolver.kt new file mode 100644 index 0000000000..5813375851 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewArtifactInfoResolver.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config + +import com.tencent.bkrepo.common.artifact.resolve.path.ArtifactInfoResolver +import com.tencent.bkrepo.common.artifact.resolve.path.Resolver +import com.tencent.bkrepo.preview.artifact.PreviewArtifactInfo +import org.springframework.stereotype.Component +import javax.servlet.http.HttpServletRequest + +@Component +@Resolver(PreviewArtifactInfo::class) +class PreviewArtifactInfoResolver : ArtifactInfoResolver { + override fun resolve( + projectId: String, + repoName: String, + artifactUri: String, + request: HttpServletRequest + ): PreviewArtifactInfo { + return PreviewArtifactInfo(projectId, repoName, artifactUri) + } +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewArtifactResourceWriter.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewArtifactResourceWriter.kt new file mode 100644 index 0000000000..fe914a4989 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewArtifactResourceWriter.kt @@ -0,0 +1,115 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config + +import com.tencent.bkrepo.common.api.exception.OverloadException +import com.tencent.bkrepo.common.api.exception.SystemErrorException +import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException +import com.tencent.bkrepo.common.artifact.metrics.RecordAbleInputStream +import com.tencent.bkrepo.common.artifact.resolve.response.AbstractArtifactResourceHandler +import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource +import com.tencent.bkrepo.common.artifact.stream.rateLimit +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.common.ratelimiter.stream.CommonRateLimitInputStream +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.storage.monitor.Throughput +import com.tencent.bkrepo.common.storage.monitor.measureThroughput +import com.tencent.bkrepo.preview.constant.PREVIEW_TMP_FILE_SAVE_PATH +import org.slf4j.LoggerFactory +import java.io.File +import java.io.IOException + +/** + * preview制品响应输出，下载到临时目录 + */ +class PreviewArtifactResourceWriter( + storageProperties: StorageProperties, + requestLimitCheckService: RequestLimitCheckService +) : AbstractArtifactResourceHandler( + storageProperties, requestLimitCheckService +) { + + private val storageProperties = storageProperties + private val requestLimitCheckService = requestLimitCheckService + + @Throws(ArtifactResponseException::class, OverloadException::class) + override fun write(resource: ArtifactResource): Throughput { + responseRateLimitCheck() + downloadRateLimitCheck(resource) + return writeArtifact(resource) + } + + private fun writeArtifact(resource: ArtifactResource): Throughput { + val request = HttpContextHolder.getRequest() + var filePath = request.getAttribute(PREVIEW_TMP_FILE_SAVE_PATH) + val inputStream = resource.getSingleStream() + val length = inputStream.range.length + var rateLimitFlag = false + var exp: Exception? = null + val recordAbleInputStream = RecordAbleInputStream(inputStream) + + try { + return measureThroughput { + val stream = requestLimitCheckService.bandwidthCheck( + recordAbleInputStream, storageProperties.response.circuitBreakerThreshold, length + ) ?: recordAbleInputStream.rateLimit( + responseRateLimitWrapper(storageProperties.response.rateLimit) + ) + rateLimitFlag = stream is CommonRateLimitInputStream + + stream.use { input -> + // 写入到临时文件 + File(filePath!!.toString()).outputStream().use { fileOutput -> + input.copyTo(fileOutput, bufferSize = getBufferSize(length)) + } + } + } + } catch (exception: IOException) { + logger.error("Failed to write artifacts to the temporary directory [$filePath]", exception) + exp = exception + throw SystemErrorException() + } catch (overloadEx: OverloadException) { + logger.error("Current limit is exceeded", overloadEx) + exp = overloadEx + throw overloadEx + } finally { + if (rateLimitFlag) { + requestLimitCheckService.bandwidthFinish(exp) + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(PreviewArtifactResourceWriter::class.java) + } +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewLocalRepository.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewLocalRepository.kt new file mode 100644 index 0000000000..6c8a20e4f0 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewLocalRepository.kt @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config + +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext +import com.tencent.bkrepo.common.artifact.repository.local.LocalRepository +import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource +import com.tencent.bkrepo.preview.constant.PREVIEW_NODE_DETAIL +import com.tencent.bkrepo.preview.constant.PreviewMessageCode +import com.tencent.bkrepo.preview.exception.PreviewNotFoundException +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component + +/** + * 公共local仓库 + */ +@Component +class PreviewLocalRepository : LocalRepository() { + + companion object { + private val logger = LoggerFactory.getLogger(PreviewLocalRepository::class.java) + } + + override fun onUpload(context: ArtifactUploadContext) { + val nodeDetail = storageManager.storeArtifactFile( + buildNodeCreateRequest(context), + context.getArtifactFile(), + context.storageCredentials + ) + context.putAttribute(PREVIEW_NODE_DETAIL, nodeDetail) + } + + override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { + val resource = super.onDownload(context) + ?: throw PreviewNotFoundException( + code = PreviewMessageCode.PREVIEW_FILE_NOT_FOUND, + "${context.artifactInfo.projectId}|${context.artifactInfo.repoName}" + + "|${context.artifactInfo.getArtifactFullPath()}" + ) + return resource + } + +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewRemoteRepository.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewRemoteRepository.kt new file mode 100644 index 0000000000..21f84d7faa --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewRemoteRepository.kt @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config + +import com.tencent.bkrepo.common.artifact.repository.remote.RemoteRepository +import org.springframework.stereotype.Component + +/** + * 公共远程仓库 + */ +@Component +class PreviewRemoteRepository : RemoteRepository() diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewVirtualRepository.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewVirtualRepository.kt new file mode 100644 index 0000000000..7be58b8eb1 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/PreviewVirtualRepository.kt @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config + +import com.tencent.bkrepo.common.artifact.repository.virtual.VirtualRepository +import org.springframework.stereotype.Component + +/** + * 公共虚拟仓库 + */ +@Component +class PreviewVirtualRepository : VirtualRepository() diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewArtifactConfigurer.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewArtifactConfigurer.kt new file mode 100644 index 0000000000..d2fb0273c5 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewArtifactConfigurer.kt @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config.configuration + +import com.tencent.bkrepo.common.artifact.config.ArtifactConfigurerSupport +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurityCustomizer +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.preview.config.PreviewLocalRepository +import com.tencent.bkrepo.preview.config.PreviewRemoteRepository +import com.tencent.bkrepo.preview.config.PreviewVirtualRepository +import org.springframework.context.annotation.Configuration + +@Configuration +class PreviewArtifactConfigurer : ArtifactConfigurerSupport() { + + override fun getRepositoryType() = RepositoryType.NONE + override fun getRepositoryTypes(): List { + return mutableListOf(RepositoryType.GENERIC) + } + override fun getLocalRepository() = SpringContextUtils.getBean() + override fun getRemoteRepository() = SpringContextUtils.getBean() + override fun getVirtualRepository() = SpringContextUtils.getBean() + + override fun getAuthSecurityCustomizer() = HttpAuthSecurityCustomizer { httpAuthSecurity -> + httpAuthSecurity.withPrefix("/preview") + .includePattern("/api/**") + } +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt new file mode 100644 index 0000000000..e9cd5e2b62 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt @@ -0,0 +1,298 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config.configuration + +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Configuration + +@Configuration +class PreviewConfig { + // 通用设置 + /** + * projectId + */ + @Value("\${preview.projectId:bk-repo}") + val projectId = "bk-repo" + + /** + * repoName + */ + @Value("\${preview.repoName:convert}") + val repoName = "convert" + + /** + * 临时文件保存路径 + */ + @Value("\${preview.file.dir:/data/workspace/tmp}") + val fileDir = "/data/workspace/tmp" + + /** + * 禁止下载的文件后缀 + */ + @Value("\${preview.prohibitSuffix:exe,dll}") + val prohibitSuffix = "exe,dll" + + /** + * 预览仓库配额,默认10G + */ + @Value("\${preview.repoQuota:10485760}") + val repoQuota: Long? = 10485760 + + /** + * 预览仓库制品保留天数,默认7天 + */ + @Value("\${preview.artifactKeepDays:7}") + val artifactKeepDays: Long? = 7 + + /** + * 预览地址缓存时间,单位s, 默认0不缓存 + */ + @Value("\${preview.fullPathCacheTime:0}") + val fullPathCacheTime: Long? = 0 + + /** + * 预览服务domain + */ + @Value("\${preview.domain}") + val domain: String? = null + + /** + * generic服务domain + */ + @Value("\${preview.genericDomain}") + val genericDomain: String? = null + + // office 相关配置 + /** + * openoffice 或 LibreOffice 的 home 路径 + */ + @Value("\${preview.office.home:}") + val officeHome: String? = null + + /** + * office 转换服务的端口，默认开启两个进程 + */ + @Value("\${preview.office.plugin.server.ports:2001,2002}") + val officePluginServerPorts = "2001,2002" + + /** + * office 转换服务 task 超时时间，默认五分钟 + */ + @Value("\${preview.office.plugin.task.timeout:5m}") + val officePluginTaskTimeout = "5m" + + /** + * 此属性设置 office 进程在重新启动之前可以执行的最大任务数。0 表示无限数量的任务（永远不会重新启动） + */ + @Value("\${preview.office.plugin.task.maxTasksPerProcess:200}") + val officePluginTaskMaxTasksPerProcess = 200 + + /** + * 此属性设置处理任务所允许的最长时间。如果任务的处理时间长于此超时，则此任务将中止，并处理下一个任务 + */ + @Value("\${preview.office.plugin.task.taskExecutionTimeout:5m}") + val officePluginTaskExecutionTimeout = "5m" + + /** + * 生成限制，默认不限，设置使用方法 (1-5) + */ + @Value("\${preview.office.pageRange:false}") + val isOfficePageRange = false + + /** + * 生成水印，默认不启用 + */ + @Value("\${preview.office.watermark:false}") + val isOfficeWatermark = false + + /** + * OFFICE JPEG 图片压缩 + */ + @Value("\${preview.office.quality:80}") + val officeQuality = 80 + + /** + * 图像分辨率限制 + */ + @Value("\${preview.office.maxImageResolution:150}") + val officeMaxImageResolution = 150 + + /** + * 导出书签 + */ + @Value("\${preview.office.exportBookmarks:true}") + val isOfficeExportBookmarks = true + + /** + * 批注作为 PDF 的注释 + */ + @Value("\${preview.office.exportNotes:true}") + val isOfficeExportNotes = true + + /** + * 加密文档生成的 PDF 文档，添加密码（密码为加密文档的密码） + */ + @Value("\${preview.office.documentOpenPasswords:true}") + val isOfficeDocumentOpenPasswords = true + + /** + * xlsx 格式前端解析 + */ + @Value("\${preview.office.type.web:web}") + val officeTypeWeb = "web" + + /** + * office 类型文档 (word, ppt) 样式，默认为pdf,也可以是图片image + */ + @Value("\${preview.office.preview.type:pdf}") + val officePreviewType = "pdf" + + /** + * 是否关闭 office 预览切换开关，默认为 false，可配置为 true 关闭 + */ + @Value("\${preview.office.preview.switch.disabled:true}") + val isOfficePreviewSwitchDisabled = true + // PDF 相关配置 + /** + * 配置 PDF 文件生成图片的像素大小，dpi 越高，图片质量越清晰，同时也会消耗更多的计算资源 + */ + @Value("\${preview.pdf2jpg.dpi:144}") + val pdfToJpgDpi = 144 + + /** + * 是否禁止演示模式 + */ + @Value("\${preview.pdf.presentationMode.disable:true}") + val isPdfPresentationModeDisable = true + + /** + * 是否禁止打开文件 + */ + @Value("\${preview.pdf.openFile.disable:true}") + val isPdfOpenFileDisable = true + + /** + * 是否禁止打印转换生成的 PDF 文件 + */ + @Value("\${preview.pdf.print.disable:true}") + val isPdfPrintDisable = true + + /** + * 是否禁止下载转换生成的 PDF 文件 + */ + @Value("\${preview.pdf.download.disable:true}") + val isPdfDownloadDisable = true + + /** + * 是否禁止 bookmarkFileConvertQueueTask + */ + @Value("\${preview.pdf.bookmark.disable:true}") + val isPdfBookmarkDisable = true + + /** + * 是否禁止签名 + */ + @Value("\${preview.pdf.disable.editing:true}") + val isPdfDisableEditing = true + // FTP配置 + /** + * ftp用户名 + */ + @Value("\${preview.ftp.username:ftpuser}") + val ftpUsername = "ftpuser" + + /** + * ftp密码 + */ + @Value("\${preview.ftp.password:123456}") + val ftpPassword = "123456" + + /** + * fFTP连接默认ControlEncoding(根据FTP服务器操作系统选择，Linux一般为UTF-8，Windows一般为GBK)， + * 可在ftp url后面加参数ftp.control.encoding=UTF-8指定，不指定默认用配置的 + */ + @Value("\${preview.ftp.controlEncoding:UTF-8}") + val ftpControlEncoding = "UTF-8" + + // 水印 + /** + * 水印内容 + */ + @Value("\${preview.watermark.txt:}") + val watermarkTxt = "" + /** + * 水印x轴间隔 + */ + @Value("\${preview.watermark.x.space:10}") + val watermarkXSpace = "10" + /** + * 水印y轴间隔 + */ + @Value("\${preview.watermark.y.space:10}") + val watermarkYSpace = "10" + /** + * 水印字体 + */ + @Value("\${preview.watermark.font:微软雅黑}") + val watermarkFont = "微软雅黑" + /** + * 水印字体大小 + */ + @Value("\${preview.watermark.fontsize:18px}") + val watermarkFontsize = "18px" + /** + * 水印颜色 + */ + @Value("\${preview.watermark.color:black}") + val watermarkColor = "black" + /** + * 水印透明度 + */ + @Value("\${preview.watermark.alpha:0.2}") + val watermarkAlpha = "0.2" + /** + * 水印宽度 + */ + @Value("\${preview.watermark.width:180}") + val watermarkWidth = "180" + /** + * 水印高度 + */ + @Value("\${preview.watermark.height:80}") + val watermarkHeight = "80" + /** + * 水印倾斜度数 + */ + @Value("\${preview.watermark.angle:10}") + val watermarkAngle = "10" + +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/ResourceWriterConfigurer.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/ResourceWriterConfigurer.kt new file mode 100644 index 0000000000..0d41353983 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/ResourceWriterConfigurer.kt @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config.configuration + +import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResourceWriter +import com.tencent.bkrepo.common.storage.config.StorageProperties +import com.tencent.bkrepo.common.ratelimiter.service.RequestLimitCheckService +import com.tencent.bkrepo.preview.config.PreviewArtifactResourceWriter +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Primary + +@Configuration +class ResourceWriterConfigurer { + + @Primary + @Bean + fun artifactResourceWriter( + storageProperties: StorageProperties, + requestLimitCheckService: RequestLimitCheckService + ): ArtifactResourceWriter { + return PreviewArtifactResourceWriter( + storageProperties, requestLimitCheckService + ) + } +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/startup/PreviewStartupRunner.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/startup/PreviewStartupRunner.kt new file mode 100644 index 0000000000..6ec82e6dbb --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/startup/PreviewStartupRunner.kt @@ -0,0 +1,115 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.config.startup + +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService +import com.tencent.bkrepo.preview.config.configuration.PreviewConfig +import com.tencent.bkrepo.repository.pojo.project.ProjectCreateRequest +import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.boot.ApplicationArguments +import org.springframework.boot.ApplicationRunner +import org.springframework.core.Ordered +import org.springframework.core.annotation.Order +import org.springframework.stereotype.Component + +/** + * 初始化文件预览需要的项目和仓库 + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +class PreviewStartupRunner( + private val config: PreviewConfig, + private val projectService: ProjectService, + private val repositoryService: RepositoryService +) : ApplicationRunner { + + companion object { + private val logger: Logger = LoggerFactory.getLogger(ApplicationRunner::class.java) + } + + override fun run(args: ApplicationArguments?) { + createProject() + createRepo() + } + + /** + * 创建仓库 + */ + private fun createRepo() { + val projectId = config.projectId + val repoName = config.repoName + var exist = repositoryService.checkExist(projectId, repoName, RepositoryType.GENERIC.name) + if (!exist) { + val repoConfig = LocalConfiguration() + val cleanupStrategy = mutableMapOf( + "enable" to true, + "cleanupType" to "retentionDays", + "cleanupValue" to config.artifactKeepDays + ) + repoConfig.settings["cleanupStrategy"] = cleanupStrategy + + val req = RepoCreateRequest(projectId = config.projectId, + name = config.repoName, + type = RepositoryType.GENERIC, + category = RepositoryCategory.LOCAL, + public = true, + quota = config.repoQuota, + configuration =repoConfig + ) + var createdRepo = repositoryService.createRepo(req) + logger.debug("Create project success，projectId:${createdRepo.name}") + } else { + logger.debug("project ${config.projectId} and repository ${config.repoName} exist skip.") + } + } + + /** + * 创建项目 + */ + private fun createProject() { + val projectId = config.projectId + var exist = projectService.checkExist(projectId); + if (!exist) { + val req = ProjectCreateRequest(name = config.projectId, displayName = config.projectId) + var createdProject = projectService.createProject(req) + logger.debug("Create project success，projectId:${createdProject.name}") + } else { + logger.debug("project ${config.projectId} exist skip.") + } + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/CommonResourceController.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/CommonResourceController.kt new file mode 100644 index 0000000000..ac817abc1c --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/CommonResourceController.kt @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.controller + +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.preview.pojo.PreviewOptions +import com.tencent.bkrepo.preview.pojo.Watermark +import com.tencent.bkrepo.preview.service.CommonResourceService +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +class CommonResourceController(private val resourceService: CommonResourceService) { + + /** + * 获取水印 + */ + @GetMapping("/api/common/getWatermark") + fun getWatermark( + @RequestParam("extraParam", required = false) extraParams: String? + ): Response { + return ResponseBuilder.success(resourceService.decodeAndParseWatermark(extraParams)) + } + + /** + * 获取预览属性 + */ + @GetMapping("/api/common/getOptions") + fun getPreviewOptions(): Response { + return ResponseBuilder.success(resourceService.getPreviewOptions()) + } + +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/FilePreviewController.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/FilePreviewController.kt new file mode 100644 index 0000000000..29f6e1be29 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/FilePreviewController.kt @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.controller + +import com.tencent.bkrepo.common.api.exception.ParameterInvalidException +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.preview.artifact.PreviewArtifactInfo +import com.tencent.bkrepo.preview.artifact.PreviewArtifactInfo.Companion.PREVIEW_COMMON_MAPPING_URI +import com.tencent.bkrepo.preview.artifact.PreviewArtifactInfo.Companion.PREVIEW_REMOTE_MAPPING_URI +import com.tencent.bkrepo.preview.constant.PreviewMessageCode +import com.tencent.bkrepo.preview.exception.PreviewInvalidException +import com.tencent.bkrepo.preview.service.FileHandlerService +import com.tencent.bkrepo.preview.service.FilePreviewFactory +import com.tencent.bkrepo.preview.utils.WebUtils +import com.tencent.bkrepo.preview.pojo.PreviewInfo +import org.slf4j.LoggerFactory +import org.springframework.http.HttpStatus +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController +import javax.servlet.http.HttpServletRequest + +/** + * 预览接口 + */ +@RestController +class FilePreviewController( + private val fileHandlerService: FileHandlerService, + private val previewFactory: FilePreviewFactory +) { + + /** + * bkrepo文件 + */ + @GetMapping(PREVIEW_COMMON_MAPPING_URI) + fun onlinePreview( + @ArtifactPathVariable artifactInfo: PreviewArtifactInfo, + @RequestParam(name = "extraParam", required = false) extraParams: String?, + req: HttpServletRequest + ): Response { + val decodedParams: String? + try { + decodedParams = if (!extraParams.isNullOrEmpty()) { + WebUtils.decodeUrl(extraParams!!).toString() + } else { + null + } + } catch (ex: Exception) { + throw PreviewInvalidException(PreviewMessageCode.PREVIEW_PARAMETER_INVALID, "extraParam") + } + val fileAttribute = fileHandlerService.getFileAttribute(artifactInfo, decodedParams, req) + val filePreview = previewFactory.get(fileAttribute) + logger.info("preview file, projectId：{},repoName:{},artifactUri:{}, previewType：{}", + artifactInfo.projectId, + artifactInfo.repoName, + artifactInfo.getArtifactFullPath(), + fileAttribute.type + ) + val previewInfo = PreviewInfo().apply { + fileName = fileAttribute.fileName + } + + filePreview.filePreviewHandle(fileAttribute, previewInfo) + return ResponseBuilder.success(previewInfo) + } + + /** + * 远程文件 + */ + @GetMapping(PREVIEW_REMOTE_MAPPING_URI) + fun onlinePreview( + @RequestParam("extraParam") extraParams: String, + req: HttpServletRequest + ): Response { + val decodedParams: String + try { + decodedParams = WebUtils.decodeUrl(extraParams)!! + } catch (ex: Exception) { + throw PreviewInvalidException(PreviewMessageCode.PREVIEW_PARAMETER_INVALID, "extraParam") + } + + val fileAttribute = fileHandlerService.getFileAttribute(decodedParams, req) + val filePreview = previewFactory.get(fileAttribute) + + logger.info("preview file, url：{}, previewType：{}", fileAttribute.url, fileAttribute.type) + + val previewInfo = PreviewInfo().apply { + fileName = fileAttribute.fileName + } + + filePreview.filePreviewHandle(fileAttribute, previewInfo) + return ResponseBuilder.success(previewInfo) + } + + + companion object { + private val logger = LoggerFactory.getLogger(FilePreviewController::class.java) + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewExceptionHandler.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewExceptionHandler.kt new file mode 100644 index 0000000000..070bf6f184 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewExceptionHandler.kt @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.exception + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.service.util.LocaleMessageUtils +import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.preview.pojo.PreviewInfo +import com.tencent.bkrepo.preview.service.FilePreview +import org.springframework.core.Ordered +import org.springframework.core.annotation.Order +import org.springframework.http.HttpStatus +import org.springframework.web.bind.annotation.ExceptionHandler +import org.springframework.web.bind.annotation.ResponseStatus +import org.springframework.web.bind.annotation.RestControllerAdvice + +/** + * 预览统一异常处理 + */ +@Order(Ordered.HIGHEST_PRECEDENCE + 1) +@RestControllerAdvice("com.tencent.bkrepo.preview") +class PreviewExceptionHandler { + /** + * 资源不存在 + */ + @ExceptionHandler(PreviewNotFoundException::class) + @ResponseStatus(HttpStatus.NOT_FOUND) + fun handleException(exception: PreviewNotFoundException) : Response { + return previewResponse(exception) + } + + /** + * 文件处理异常 + */ + @ExceptionHandler(PreviewHandleException::class) + @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) + fun handleException(exception: PreviewHandleException) : Response { + return previewResponse(exception) + } + + /** + * 请求参数异常 + */ + @ExceptionHandler(PreviewInvalidException::class) + @ResponseStatus(HttpStatus.BAD_REQUEST) + fun handleException(exception: PreviewInvalidException) : Response { + return previewResponse(exception) + } + + private fun previewResponse(exception: ErrorCodeException) : Response { + val previewInfo = PreviewInfo() + val errorMessage = LocaleMessageUtils.getLocalizedMessage(exception.messageCode, exception.params) + previewInfo.fileTemplate = FilePreview.NOT_SUPPORTED_FILE_PAGE + previewInfo.msg = errorMessage + return ResponseBuilder.build(exception.messageCode.getCode(), errorMessage ,previewInfo) + } +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewHandleException.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewHandleException.kt new file mode 100644 index 0000000000..4e7faddff3 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewHandleException.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.exception + +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.MessageCode +import com.tencent.bkrepo.preview.constant.PreviewMessageCode + +/** + * 文件处理异常 + */ +open class PreviewHandleException( + messageCode: MessageCode = PreviewMessageCode.PREVIEW_FIlE_HANDLE_ERROR, + param: String +) : ErrorCodeException(HttpStatus.INTERNAL_SERVER_ERROR, messageCode, arrayOf(param)) \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewInvalidException.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewInvalidException.kt new file mode 100644 index 0000000000..5c08eb2ae0 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewInvalidException.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.exception + +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.MessageCode + +/** + * 参数非法异常 + */ +class PreviewInvalidException( + code: MessageCode, + paramName: String +) : ErrorCodeException(code, paramName) diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewNotFoundException.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewNotFoundException.kt new file mode 100644 index 0000000000..09e89a0cd6 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/exception/PreviewNotFoundException.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.exception + +import com.tencent.bkrepo.common.api.exception.NotFoundException +import com.tencent.bkrepo.common.api.message.MessageCode + +/** + * 文件预览，文件不存在 + */ +open class PreviewNotFoundException( + code: MessageCode, + param: String +) : NotFoundException(code, param) \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/CommonResourceService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/CommonResourceService.kt new file mode 100644 index 0000000000..303b8dd2c2 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/CommonResourceService.kt @@ -0,0 +1,65 @@ +package com.tencent.bkrepo.preview.service + +import com.tencent.bkrepo.preview.config.configuration.PreviewConfig +import com.tencent.bkrepo.preview.constant.PreviewMessageCode +import com.tencent.bkrepo.preview.exception.PreviewInvalidException +import com.tencent.bkrepo.preview.pojo.PreviewOptions +import com.tencent.bkrepo.preview.pojo.Watermark +import com.tencent.bkrepo.preview.utils.JsonMapper +import com.tencent.bkrepo.preview.utils.WebUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component + +@Component +class CommonResourceService(private val config: PreviewConfig) { + + companion object { + private val logger: Logger = LoggerFactory.getLogger(CommonResourceService::class.java) + } + + /** + * 水印 + */ + fun decodeAndParseWatermark(params: String?): Watermark { + var watermark:Watermark + val jsonMapper = JsonMapper() + try { + watermark = if (!params.isNullOrEmpty()) { + val decodedParams = WebUtils.decodeUrl(params).toString() + jsonMapper.fromJson(decodedParams, Watermark::class.java)?: Watermark() + } else { + Watermark() + } + } catch (ex: Exception) { + logger.warn("extraParam decoding error.", ex) + throw PreviewInvalidException(PreviewMessageCode.PREVIEW_PARAMETER_INVALID, "extraParam") + } + watermark.watermarkTxt = watermark.watermarkTxt ?: config.watermarkTxt + watermark.watermarkXSpace = watermark.watermarkXSpace ?: config.watermarkXSpace + watermark.watermarkYSpace = watermark.watermarkYSpace ?: config.watermarkYSpace + watermark.watermarkFont = watermark.watermarkFont ?: config.watermarkFont + watermark.watermarkFontsize = watermark.watermarkFontsize ?: config.watermarkFontsize + watermark.watermarkColor = watermark.watermarkColor ?: config.watermarkColor + watermark.watermarkAlpha = watermark.watermarkAlpha ?: config.watermarkAlpha + watermark.watermarkWidth = watermark.watermarkWidth ?: config.watermarkWidth + watermark.watermarkHeight = watermark.watermarkHeight ?: config.watermarkHeight + watermark.watermarkAngle = watermark.watermarkAngle ?: config.watermarkAngle + return watermark + } + + /** + * 预览属性 + */ + fun getPreviewOptions(): PreviewOptions { + return PreviewOptions( + pdfPresentationModeDisable = config.isPdfPresentationModeDisable, + pdfOpenFileDisable = config.isPdfOpenFileDisable, + pdfPrintDisable = config.isPdfPrintDisable, + pdfDownloadDisable = config.isPdfDownloadDisable, + pdfBookmarkDisable = config.isPdfBookmarkDisable, + pdfDisableEditing = config.isPdfDisableEditing, + switchDisabled = config.isOfficePreviewSwitchDisabled + ) + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt new file mode 100644 index 0000000000..b421bfa441 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt @@ -0,0 +1,285 @@ +package com.tencent.bkrepo.preview.service + +import com.tencent.bkrepo.common.api.exception.ParameterInvalidException +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.preview.config.configuration.PreviewConfig +import com.tencent.bkrepo.preview.constant.PreviewMessageCode +import com.tencent.bkrepo.preview.exception.PreviewInvalidException +import com.tencent.bkrepo.preview.utils.* +import com.tencent.bkrepo.preview.pojo.FileAttribute +import com.tencent.bkrepo.preview.pojo.FileType +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component +import org.springframework.util.StringUtils +import java.io.* +import java.net.URLDecoder +import java.net.URLEncoder +import java.nio.charset.Charset +import java.util.* +import javax.servlet.http.HttpServletRequest + +@Component +class FileHandlerService(private val config: PreviewConfig) { + + companion object { + private const val URI_ENCODING = "UTF-8" + private const val PDF2JPG_IMAGE_FORMAT = ".jpg" + private val logger = LoggerFactory.getLogger(FileHandlerService::class.java) + } + + /** + * 对转换后的文件进行操作(改变编码方式) + * + * @param outFilePath 文件绝对路径 + */ + fun doActionConvertedFile(outFilePath: String) { + val charset = EncodingDetects.getJavaEncode(outFilePath) // 假设返回的是 String? + val sb = StringBuilder() + try { + FileInputStream(outFilePath).bufferedReader(charset = Charset.forName(charset)).use { reader -> + var line: String? + while (reader.readLine().also { line = it } != null) { + if (line!!.contains("charset=gb2312")) { + line = line!!.replace("charset=gb2312", "charset=utf-8") + } + sb.append(line) + } + // 添加sheet控制头 + sb.append("") + sb.append("") + sb.append("") + } + } catch (e: IOException) { + e.printStackTrace() + } + // 重新写入文件 + try { + FileOutputStream(outFilePath).bufferedWriter(Charset.forName("UTF-8")).use { writer -> + writer.write(sb.toString()) + } + } catch (e: IOException) { + e.printStackTrace() + } + } + + /** + * 获取本地 pdf 转 image 后的 web 访问地址 + * + * @param pdfFilePath pdf文件名 + * @param index 图片索引 + * @return 图片访问地址 + */ + private fun getPdf2jpgUrl(pdfFilePath: String, index: Int): String { + val baseUrl = "" // 设置 baseUrl + val pdfFolder = pdfFilePath.replace(config.fileDir, "").removeSuffix(".pdf") + val urlPrefix: String + urlPrefix = try { + "$baseUrl${URLEncoder.encode(pdfFolder, URI_ENCODING).replace("+", "%20")}" + } catch (e: UnsupportedEncodingException) { + logger.error("UnsupportedEncodingException", e) + "$baseUrl$pdfFolder" + } + return "$urlPrefix/$index$PDF2JPG_IMAGE_FORMAT" + } + + /** + * @param str 原字符串（待截取原串） + * @param posStr 指定字符串 + * @return 截取指定字符串之后的数据 + */ + fun getSubString(str: String, posStr: String): String { + return str.substring(str.indexOf(posStr) + posStr.length) + } + + /** + * 获取文件属性 + * + * @param params 原始参数 + * @return 文件属性 + */ + fun getFileAttribute(params: String, req: HttpServletRequest): FileAttribute { + val jsonMapper = JsonMapper() + val attribute = jsonMapper.fromJson(params, FileAttribute::class.java)?: FileAttribute() + checkRequest(attribute, req) + adjustProperties(attribute, req) + return attribute + } + + /** + * 获取文件属性 + * + * @param params 原始参数 + * @return 文件属性 + */ + fun getFileAttribute(artifactInfo: ArtifactInfo, params: String?, req: HttpServletRequest): FileAttribute { + val jsonMapper = JsonMapper() + val attribute = jsonMapper.fromJson(params, FileAttribute::class.java)?: FileAttribute() + attribute.projectId = artifactInfo.projectId + attribute.repoName = artifactInfo.repoName + attribute.artifactUri = artifactInfo.getArtifactFullPath() + checkRequest(attribute, req) + adjustProperties(attribute, req) + return attribute + } + + /** + * 获取转换的文件名 + * + * @return 文件名 + */ + private fun getConvertFileName( + type: FileType, + originFileName: String, + convertFilePrefixName: String, + isHtmlView: Boolean, + isCompressFile: Boolean + ): String { + var convertFileName = when (type) { + FileType.OFFICE -> convertFilePrefixName + if (isHtmlView) "html" else "pdf" + FileType.PDF -> originFileName + FileType.MEDIACONVERT -> convertFilePrefixName + "mp4" + else -> originFileName + } + if (isCompressFile) { // 判断是否使用特定压缩包符号 + convertFileName = "_decompression$convertFileName" + } + return convertFileName + } + + private fun getBaseUrl(request: HttpServletRequest): String { + // 1. 支持通过 http header 中 X-Base-Url 来动态设置 baseUrl 以支持多个域名/项目的共享使用 + val urlInHeader = request.getHeader("X-Base-Url") + var baseUrl: String + + baseUrl = if (!urlInHeader.isNullOrEmpty()) { + urlInHeader + } else (if (!config.domain.isNullOrEmpty()) { + // 2. 如果配置文件中配置了 baseUrl 且不为 default 则以配置文件为准 + config.domain + } else { + // 3. 默认动态拼接 baseUrl + "${request.scheme}://${request.serverName}:${request.serverPort}${request.contextPath}/" + }).toString() + + if (!baseUrl.endsWith("/")) { + baseUrl = "$baseUrl/" + } + + return baseUrl + } + + + + /** + * 请求校验 + */ + private fun checkRequest(fileAttribute: FileAttribute, req: HttpServletRequest) { + if (fileAttribute.projectId.isNullOrEmpty() + && fileAttribute.repoName.isNullOrEmpty() + && fileAttribute.artifactUri.isNullOrEmpty() + && fileAttribute.url.isNullOrEmpty()) { + logger.error("The file download info cannot be empty") + throw PreviewInvalidException(PreviewMessageCode.PREVIEW_PARAMETER_INVALID, "url") + } + } + + /** + * 参数完善 + */ + private fun adjustProperties(fileAttribute: FileAttribute, req: HttpServletRequest) { + var suffix: String + var type: FileType + var originFileName: String // 原始文件名 + val fullFileName = fileAttribute.fileName + var url = fileAttribute.url + + // 设置 storageType + fileAttribute.storageType = determineStorageType(fileAttribute) + + // 判断文件名和类型 + if (!fullFileName.isNullOrBlank()) { + originFileName = fullFileName + type = FileType.typeFromFileName(fullFileName) + suffix = FileUtils.suffixFromFileName(fullFileName) + } else { + if (fileAttribute.storageType == 0) { + originFileName = WebUtils.getFileNameFromURL(fileAttribute.artifactUri!!) + type = FileType.typeFromUrl(fileAttribute.artifactUri!!) + suffix = WebUtils.suffixFromUrl(fileAttribute.artifactUri!!) + } else { + originFileName = WebUtils.getFileNameFromURL(url!!) + type = FileType.typeFromUrl(url) + suffix = WebUtils.suffixFromUrl(url) + } + } + + // 判断文件名是否转义 + if (UrlEncoderUtils.hasUrlEncoded(originFileName)) { + try { + originFileName = URLDecoder.decode(originFileName, URI_ENCODING) + } catch (e: UnsupportedEncodingException) { + e.printStackTrace() + logger.error("File name [$originFileName] escaping error.", e) + } + } else { + url = url?.let { WebUtils.encodeUrlFileName(it) } // 对未转义的url进行转义 + } + + // 文件名处理 + originFileName = FileUtils.htmlEscape(originFileName) + + // 转换后的文件名前缀 + var convertFilePrefixName: String? = null + try { + convertFilePrefixName = originFileName.substring(0, originFileName.lastIndexOf(".")) + suffix + "." + } catch (e: Exception) { + logger.error("Get file name suffix incorrectly：", e) + } + + // 判断是否为 HTML 视图 + val isHtmlView = isHtmlView(suffix) + + // 获取缓存文件名 + val convertFileName = convertFilePrefixName?.let { + getConvertFileName(type, originFileName, it, isHtmlView, false) + } + + // 获取输出文件路径 + val outFilePath = generateOutputFilePath(convertFileName) + + // 设置 FileAttribute 属性 + fileAttribute.apply { + this.type = type + this.fileName = originFileName + this.suffix = suffix + this.url = url + this.outFilePath = outFilePath + this.convertFileName = convertFileName + this.isHtmlView = isHtmlView + this.baseUrl = getBaseUrl(req) + } + } + + // 生成输出文件路径 + private fun generateOutputFilePath(convertFileName: String?): String { + return "${config.fileDir}${File.separator}convert${File.separator}" + + "${UUID.randomUUID()}${File.separator}$convertFileName" + } + + // 解析 storageType + private fun determineStorageType(fileAttribute: FileAttribute): Int { + return if (!fileAttribute.projectId.isNullOrBlank() && + !fileAttribute.repoName.isNullOrBlank() && + !fileAttribute.artifactUri.isNullOrBlank()) { + 0 + } else { + 1 + } + } + + // 判断是否为 HTML 视图 + private fun isHtmlView(suffix: String): Boolean { + val htmlSuffixes = setOf("xls", "xlsx", "csv", "xlsm", "xlt", "xltm", "et", "ett", "xlam") + return htmlSuffixes.contains(suffix.lowercase()) + } +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreview.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreview.kt new file mode 100644 index 0000000000..697ae3338f --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreview.kt @@ -0,0 +1,35 @@ +package com.tencent.bkrepo.preview.service + +import com.tencent.bkrepo.preview.pojo.FileAttribute +import com.tencent.bkrepo.preview.pojo.PreviewInfo + +interface FilePreview { + fun filePreviewHandle(fileAttribute: FileAttribute, previewInfo: PreviewInfo): PreviewInfo? + + companion object { + const val PDF_FILE_PREVIEW_PAGE = "pdf" + const val PPT_FILE_PREVIEW_PAGE = "ppt" + const val COMPRESS_FILE_PREVIEW_PAGE = "compress" + const val MEDIA_FILE_PREVIEW_PAGE = "media" + const val PICTURE_FILE_PREVIEW_PAGE = "picture" + const val TIFF_FILE_PREVIEW_PAGE = "tiff" + const val OFD_FILE_PREVIEW_PAGE = "ofd" + const val SVG_FILE_PREVIEW_PAGE = "svg" + const val ONLINE3D_PREVIEW_PAGE = "online3D" + const val EPUB_PREVIEW_PAGE = "epub" + const val XMIND_FILE_PREVIEW_PAGE = "xmind" + const val EML_FILE_PREVIEW_PAGE = "eml" + const val OFFICE_PICTURE_FILE_PREVIEW_PAGE = "officePicture" + const val TXT_FILE_PREVIEW_PAGE = "txt" + const val CODE_FILE_PREVIEW_PAGE = "code" + const val EXEL_FILE_PREVIEW_PAGE = "html" + const val XML_FILE_PREVIEW_PAGE = "xml" + const val MARKDOWN_FILE_PREVIEW_PAGE = "markdown" + const val BPMN_FILE_PREVIEW_PAGE = "bpmn" + const val DCM_FILE_PREVIEW_PAGE = "dcm" + const val DRAWUI_FILE_PREVIEW_PAGE = "drawio" + const val NOT_SUPPORTED_FILE_PAGE = "fileNotSupported" + const val XLSX_FILE_PREVIEW_PAGE = "officeweb" + const val CSV_FILE_PREVIEW_PAGE = "csv" + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreviewFactory.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreviewFactory.kt new file mode 100644 index 0000000000..c193eeb766 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreviewFactory.kt @@ -0,0 +1,14 @@ +package com.tencent.bkrepo.preview.service + +import com.tencent.bkrepo.preview.pojo.FileAttribute +import com.tencent.bkrepo.preview.pojo.FileType +import com.tencent.bkrepo.preview.service.FilePreview +import org.springframework.context.ApplicationContext +import org.springframework.stereotype.Service + +@Service +class FilePreviewFactory(private val context: ApplicationContext) { + operator fun get(fileAttribute: FileAttribute): FilePreview { + return context.getBean(fileAttribute.type!!.instanceName, FilePreview::class.java) + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileTransferService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileTransferService.kt new file mode 100644 index 0000000000..69631d0966 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileTransferService.kt @@ -0,0 +1,149 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.service + +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.constant.ARTIFACT_INFO_KEY +import com.tencent.bkrepo.common.artifact.constant.REPO_KEY +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException +import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException +import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext +import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext +import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService +import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactFileFactory +import com.tencent.bkrepo.common.metadata.service.project.ProjectService +import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.preview.config.configuration.PreviewConfig +import com.tencent.bkrepo.preview.constant.PREVIEW_NODE_DETAIL +import com.tencent.bkrepo.preview.constant.PREVIEW_TMP_FILE_SAVE_PATH +import com.tencent.bkrepo.preview.constant.PreviewMessageCode +import com.tencent.bkrepo.preview.exception.PreviewNotFoundException +import com.tencent.bkrepo.preview.pojo.DownloadResult +import com.tencent.bkrepo.preview.pojo.FileAttribute +import com.tencent.bkrepo.preview.utils.DownloadUtils +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import org.springframework.stereotype.Component +import java.io.File +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.util.* +import javax.servlet.http.HttpServletRequest + +@Component +class FileTransferService( + private val config: PreviewConfig, + private val projectService: ProjectService, + private val repositoryService: RepositoryService + ) : ArtifactService() { + fun download(fileAttribute: FileAttribute): DownloadResult? { + var result: DownloadResult? = if (fileAttribute.storageType == 1) { + DownloadUtils.downLoad(fileAttribute, config) + } else { + downloadFromRepo(fileAttribute) + } + return result + } + + fun downloadFromRepo(fileAttribute: FileAttribute):DownloadResult? { + val artifactInfo = ArtifactInfo(fileAttribute.projectId!!, + fileAttribute.repoName!!, + fileAttribute.artifactUri!! + ) + setFileTransferAttribute(artifactInfo, fileAttribute) + val node = ArtifactContextHolder.getNodeDetail(artifactInfo) + val context = ArtifactDownloadContext() + if (node == null && context.repositoryDetail.category == RepositoryCategory.LOCAL) { + throw PreviewNotFoundException(PreviewMessageCode.PREVIEW_NODE_NOT_FOUND, + "${artifactInfo.projectId}|${artifactInfo.repoName}|${artifactInfo.getArtifactFullPath()}") + } + val result = DownloadResult() + try { + repository.download(context) + val request: HttpServletRequest = HttpContextHolder.getRequest() + result.filePath = request.getAttribute(PREVIEW_TMP_FILE_SAVE_PATH)?.toString() + result.md5 = node!!.md5 + result.size = node.size + } catch (e: Exception) { + result.apply { + code = DownloadResult.CODE_FAIL + msg = "Download Faile from bkrepo,$e" + } + } + + return result + } + + fun upload(fileAttribute: FileAttribute, sourcePath: String): NodeDetail{ + val file = File(sourcePath) + require(file.exists()) { "The file does not exist, $sourcePath" } + // 准备要上传的信息 + val artifactInfo = ArtifactInfo(config.projectId, config.repoName, buildArtifactUri(fileAttribute)) + setFileTransferAttribute(artifactInfo, fileAttribute) + val artifactFile = ArtifactFileFactory.build(file.inputStream(), file.length()) + val context = ArtifactUploadContext(artifactFile) + + repository.upload(context) + + val nodeDetail: NodeDetail = context.getAttribute(PREVIEW_NODE_DETAIL)!! + return nodeDetail + } + + /** + * 把project、repo等信息设置到request域，上传、下载要用 + */ + private fun setFileTransferAttribute(artifactInfo: ArtifactInfo, fileAttribute: FileAttribute) { + val request: HttpServletRequest = HttpContextHolder.getRequest() + val repoDetail =repositoryService.getRepoDetail(artifactInfo.projectId, + artifactInfo.repoName, + RepositoryType.GENERIC.name + ) ?: throw PreviewNotFoundException(PreviewMessageCode.PREVIEW_REPO_NOT_FOUND, + "${artifactInfo.projectId}|${artifactInfo.repoName}") + request.setAttribute(ARTIFACT_INFO_KEY, artifactInfo) + request.setAttribute(REPO_KEY, repoDetail) + + val fileTmpPath = DownloadUtils.getRelFilePath(fileAttribute.fileName, fileAttribute.suffix!!, config.fileDir) + request.setAttribute(PREVIEW_TMP_FILE_SAVE_PATH, fileTmpPath) + } + + /** + * 制品uri：/date/uuid/fileName + */ + private fun buildArtifactUri(fileAttribute: FileAttribute): String { + val date = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")) + return "/$date/${UUID.randomUUID()}/${fileAttribute.convertFileName}" + } + +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt new file mode 100644 index 0000000000..5ca09a68c0 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt @@ -0,0 +1,168 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.service + +import org.apache.commons.lang3.StringUtils +import org.jodconverter.core.office.InstalledOfficeManagerHolder +import org.jodconverter.core.office.OfficeException +import org.jodconverter.core.office.OfficeUtils +import org.jodconverter.core.util.OSUtils +import org.jodconverter.local.office.LocalOfficeManager +import org.jodconverter.local.office.LocalOfficeUtils +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.convert.DurationStyle +import org.springframework.stereotype.Component +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.IOException +import java.util.* +import java.util.function.ToIntFunction +import javax.annotation.PostConstruct +import javax.annotation.PreDestroy + +/** + * 创建office文件转换器 + */ +@Component +class OfficePluginManager { + private lateinit var officeManager: LocalOfficeManager + + @Value("\${preview.office.plugin.server.ports:2001,2002}") + private val serverPorts = "2001,2002" + + @Value("\${preview.office.plugin.task.timeout:5m}") + private val timeOut = "5m" + + @Value("\${preview.office.plugin.task.taskExecutionTimeout:5m}") + private val taskExecutionTimeout = "5m" + + @Value("\${preview.office.plugin.task.maxTasksPerProcess:5}") + private val maxTasksPerProcess = 5 + + /** + * 启动Office组件进程 + */ + @PostConstruct + fun startOfficeManager() { + val officeHome: File = LocalOfficeUtils.getDefaultOfficeHome() + val killOffice = killProcess() + if (killOffice) { + logger.warn("A running Office process has been detected and the process has been automatically terminated") + } + try { + val portsString = serverPorts!!.split(",".toRegex()).toTypedArray() + val ports = portsString.map { it.toInt() }.toIntArray() + val timeout = DurationStyle.detectAndParse(timeOut).toMillis() + val taskexecutiontimeout = DurationStyle.detectAndParse(taskExecutionTimeout).toMillis() + officeManager = LocalOfficeManager.builder() + .officeHome(officeHome) + .portNumbers(*ports) + .processTimeout(timeout) + .maxTasksPerProcess(maxTasksPerProcess) + .taskExecutionTimeout(taskexecutiontimeout) + .build() + officeManager.start() + InstalledOfficeManagerHolder.setInstance(officeManager) + } catch (e: Exception) { + logger.error("The office component fails to start, check whether the office component is available",e) + } + } + + private fun killProcess(): Boolean { + var flag = false + try { + if (OSUtils.IS_OS_WINDOWS) { + val p = Runtime.getRuntime().exec("cmd /c tasklist ") + val baos = ByteArrayOutputStream() + val os = p.inputStream + val b = ByteArray(256) + while (os.read(b) > 0) { + baos.write(b) + } + val s = baos.toString() + if (s.contains("soffice.bin")) { + Runtime.getRuntime().exec("taskkill /im " + "soffice.bin" + " /f") + flag = true + } + } else if (OSUtils.IS_OS_MAC || OSUtils.IS_OS_MAC_OSX) { + val p = Runtime.getRuntime().exec(arrayOf("sh", "-c", "ps -ef | grep " + "soffice.bin")) + val baos = ByteArrayOutputStream() + val os = p.inputStream + val b = ByteArray(256) + while (os.read(b) > 0) { + baos.write(b) + } + val s = baos.toString() + if (StringUtils.ordinalIndexOf(s, "soffice.bin", 3) > 0) { + val cmd = arrayOf("sh", "-c", "kill -15 `ps -ef|grep " + "soffice.bin" + "|awk 'NR==1{print $2}'`") + Runtime.getRuntime().exec(cmd) + flag = true + } + } else { + val p = Runtime.getRuntime() + .exec(arrayOf("sh", "-c", "ps -ef | grep " + "soffice.bin" + " |grep -v grep | wc -l")) + val baos = ByteArrayOutputStream() + val os = p.inputStream + val b = ByteArray(256) + while (os.read(b) > 0) { + baos.write(b) + } + val s = baos.toString() + if (!s.startsWith("0")) { + val cmd = arrayOf( + "sh", + "-c", + "ps -ef | grep soffice.bin | grep -v grep | awk '{print \"kill -9 \"$2}' | sh" + ) + Runtime.getRuntime().exec(cmd) + flag = true + } + } + } catch (e: IOException) { + logger.error("Detect office process exceptions", e) + } + return flag + } + + @PreDestroy + fun destroyOfficeManager() { + if (null != officeManager && officeManager.isRunning()) { + logger.info("Shutting down office process") + OfficeUtils.stopQuietly(officeManager) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(OfficePluginManager::class.java) + } +} diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficeToPdfService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficeToPdfService.kt new file mode 100644 index 0000000000..a8e5cf50a2 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficeToPdfService.kt @@ -0,0 +1,122 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.service + +import com.tencent.bkrepo.common.api.exception.SystemErrorException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.preview.config.configuration.PreviewConfig +import com.tencent.bkrepo.preview.pojo.FileAttribute +import org.jodconverter.core.office.OfficeException +import org.jodconverter.local.LocalConverter +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component +import java.io.File + +/** + * office转pdf服务 + */ +@Component +class OfficeToPdfService(private val config: PreviewConfig) { + + companion object { + private val logger: Logger = LoggerFactory.getLogger(OfficeToPdfService::class.java) + } + + @Throws(OfficeException::class) + fun openOfficeToPDF(inputFilePath: String, outputFilePath: String, fileAttribute: FileAttribute) { + office2pdf(inputFilePath, outputFilePath, fileAttribute) + } + + @Throws(OfficeException::class) + fun converterFile(inputFile: File, outputFilePathEnd: String, fileAttribute: FileAttribute) { + val outputFile = File(outputFilePathEnd) + + // 如果目标目录不存在，则尝试创建 + if (!outputFile.parentFile.exists() && !outputFile.parentFile.mkdirs()) { + logger.error("Failed to create directory [$outputFilePathEnd], please check the directory permissions!") + throw SystemErrorException(CommonMessageCode.SYSTEM_ERROR, "Failed to create directory") + } + + // 设置转换文件时的过滤条件 + val filterData = mutableMapOf( + "EncryptFile" to true, // 加密文件 + "PageRange" to config.isOfficePageRange, // 限制页面范围 + "Watermark" to config.isOfficeWatermark, // 水印 + "Quality" to config.officeQuality, // 图片压缩质量 + "MaxImageResolution" to config.officeMaxImageResolution, // DPI + "ExportBookmarks" to config.isOfficeExportBookmarks, // 导出书签 + "ExportNotes" to config.isOfficeExportNotes // 导出批注作为PDF注释 + ) + + // 自定义属性，包含过滤条件 + val customProperties = mutableMapOf( + "FilterData" to filterData + ) + + // 使用 LocalConverter 构建转换器 + val builder = LocalConverter.builder().storeProperties(customProperties) + + // 执行文件转换 + builder.build().convert(inputFile).to(outputFile).execute() + } + + @Throws(OfficeException::class) + fun office2pdf(inputFilePath: String?, outputFilePath: String?, fileAttribute: FileAttribute) { + inputFilePath?.let { + val inputFile = File(it) + + // 如果目标文件路径为空，则使用默认的输出路径 + if (outputFilePath == null) { + val outputFilePathEnd = getOutputFilePath(it) + if (inputFile.exists()) { + converterFile(inputFile, outputFilePathEnd, fileAttribute) + } + } else { + // 如果目标路径不为空，直接进行转换 + if (inputFile.exists()) { + converterFile(inputFile, outputFilePath, fileAttribute) + } + } + } + } + + // 私有方法，获取输出文件路径，将输入文件路径中的扩展名改为 .pdf + private fun getOutputFilePath(inputFilePath: String): String { + return inputFilePath.replace(".${getPostfix(inputFilePath)}", ".pdf") + } + + // 私有方法，获取文件扩展名 + private fun getPostfix(inputFilePath: String): String { + return inputFilePath.substring(inputFilePath.lastIndexOf(".") + 1) + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/OfficeFilePreviewImpl.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/OfficeFilePreviewImpl.kt new file mode 100644 index 0000000000..f80c8a0fe2 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/OfficeFilePreviewImpl.kt @@ -0,0 +1,169 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.service.impl + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.redis.RedisOperation +import com.tencent.bkrepo.preview.config.configuration.PreviewConfig +import com.tencent.bkrepo.preview.constant.PreviewMessageCode +import com.tencent.bkrepo.preview.exception.PreviewHandleException +import com.tencent.bkrepo.preview.exception.PreviewNotFoundException +import com.tencent.bkrepo.preview.pojo.DownloadResult +import com.tencent.bkrepo.preview.pojo.FileAttribute +import com.tencent.bkrepo.preview.pojo.PreviewInfo +import com.tencent.bkrepo.preview.service.FilePreview +import com.tencent.bkrepo.preview.service.FileTransferService +import com.tencent.bkrepo.preview.service.OfficeToPdfService +import org.jodconverter.core.office.OfficeException +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import org.springframework.util.StringUtils + +/** + * 处理office文件 + */ +@Service +class OfficeFilePreviewImpl( + private val config: PreviewConfig, + private val officeToPdfService: OfficeToPdfService, + private val otherFilePreview: OtherFilePreviewImpl, + private val fileTransferService: FileTransferService, + private val redisOperation: RedisOperation +) : FilePreview { + + override fun filePreviewHandle(fileAttribute: FileAttribute, previewInfo: PreviewInfo): PreviewInfo { + val outFilePath = fileAttribute.outFilePath + //根据md5优先在缓存中匹配，如果缓存中没有，需要重新下载、转换文件 + var fileUri:String? = getFileUriFromCache(fileAttribute.md5) + if (fileUri.isNullOrEmpty()) { + + val downloadResult = downloadFile(fileAttribute) + ?: throw PreviewNotFoundException(PreviewMessageCode.PREVIEW_FILE_NOT_FOUND, fileAttribute.fileName!!) + + fileUri = getFileUriFromCache(downloadResult.md5) + ?: processFileConversion(downloadResult, fileAttribute, outFilePath) + } + setPreviewInfo(previewInfo, fileUri, fileAttribute) + return previewInfo + } + + private fun processFileConversion(downloadResult: DownloadResult, + fileAttribute: FileAttribute, + outFilePath: String?): String { + val filePath = downloadResult.filePath ?: return "" + var fileUri: String + try { + fileUri = if (isNeedConvert(fileAttribute)) { + officeToPdfService.openOfficeToPDF(filePath, outFilePath!!, fileAttribute) + fileTransferService.upload(fileAttribute, outFilePath).fullPath + } else { + fileTransferService.upload(fileAttribute, filePath).fullPath + } + // 更新缓存 + setFileUriCache(downloadResult.md5!!, fileUri, config.fullPathCacheTime) + } catch (e: OfficeException) { + logger.error("Failed to convert office file", e) + throw PreviewHandleException(PreviewMessageCode.PREVIEW_FIlE_HANDLE_ERROR, fileAttribute.fileName!!) + } + return fileUri + } + + private fun downloadFile(fileAttribute: FileAttribute): DownloadResult? { + val result = fileTransferService.download(fileAttribute) + if (result?.code == DownloadResult.CODE_FAIL) { + logger.error("File download failed for file: ${fileAttribute.fileName}. Error message: ${result.msg}") + return null + } + return result + } + + /** + * 设置文件预览信息 + */ + private fun setPreviewInfo(previewInfo: PreviewInfo, + fileUri: String, + fileAttribute: FileAttribute) { + when (fileAttribute.suffix!!.lowercase()) { + "xlsx" -> { + previewInfo.pdfUrl = buildPreviewUrl(fileUri, fileAttribute) + previewInfo.fileTemplate = FilePreview.XLSX_FILE_PREVIEW_PAGE + previewInfo.msg = "Previewing XLSX file" + } + "csv" -> { + previewInfo.url = buildPreviewUrl(fileUri, fileAttribute) + previewInfo.fileTemplate = FilePreview.CSV_FILE_PREVIEW_PAGE + previewInfo.msg = "Previewing CSV file" + } + else -> { + previewInfo.fileName = fileAttribute.fileName + previewInfo.pdfUrl = buildPreviewUrl(fileUri, fileAttribute) + previewInfo.fileTemplate = FilePreview.PDF_FILE_PREVIEW_PAGE + previewInfo.msg = "Previewing PDF file" + } + } + } + + /** + * 设置缓存路径 + */ + private fun setFileUriCache(fileMd5: String, fileUri: String, expiredInSecond: Long?) { + if (expiredInSecond != null && expiredInSecond > 0) { + redisOperation.set("$KEY_PREVIEW_FILE_PATH_PREFIX${StringPool.COLON}$fileMd5", fileUri, expiredInSecond) + } + } + + /** + * 获取缓存路径 + */ + private fun getFileUriFromCache(fileMd5: String?): String? { + return if (StringUtils.hasText(fileMd5)) { + redisOperation.get("$KEY_PREVIEW_FILE_PATH_PREFIX${StringPool.COLON}${fileMd5}") + } else null + } + + private fun isNeedConvert(fileAttribute: FileAttribute): Boolean { + val suffix = fileAttribute.suffix + return suffix!!.lowercase() !in listOf("xlsx", "csv") + } + + private fun buildPreviewUrl(uri: String, fileAttribute: FileAttribute): String { + val newUri = "${config.projectId}/${config.repoName}$uri" + val baseUrl = config.genericDomain ?: fileAttribute.baseUrl + val formattedBaseUrl = if (baseUrl?.endsWith("/") == true) baseUrl else "$baseUrl/" + return "$formattedBaseUrl$newUri" + } + + companion object { + private val logger = LoggerFactory.getLogger(OfficeFilePreviewImpl::class.java) + private const val KEY_PREVIEW_FILE_PATH_PREFIX = "preview:file:fullpath" + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/OtherFilePreviewImpl.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/OtherFilePreviewImpl.kt new file mode 100644 index 0000000000..95745aa12e --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/OtherFilePreviewImpl.kt @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.service.impl + +import com.tencent.bkrepo.preview.pojo.FileAttribute +import com.tencent.bkrepo.preview.utils.FileUtils.htmlEscape +import com.tencent.bkrepo.preview.service.FilePreview +import com.tencent.bkrepo.preview.pojo.PreviewInfo +import org.springframework.stereotype.Service + +@Service +class OtherFilePreviewImpl : FilePreview { + override fun filePreviewHandle(fileAttribute: FileAttribute, previewInfo: PreviewInfo): PreviewInfo { + return this.notSupportedFile(fileAttribute, + previewInfo, + "The system does not yet support online preview of files in this format" + ) + } + + /** + * 通用的预览失败，导向到不支持的文件响应页面 + * + * @return 页面 + */ + fun notSupportedFile(fileAttribute: FileAttribute, previewInfo: PreviewInfo, errMsg: String?): PreviewInfo { + return this.notSupportedFile(previewInfo, fileAttribute.suffix!!, errMsg) + } + + /** + * 通用的预览失败，导向到不支持的文件响应页面 + * + * @return 页面 + */ + fun notSupportedFile(previewInfo: PreviewInfo, errMsg: String?): PreviewInfo { + return this.notSupportedFile(previewInfo, "Unknown", errMsg) + } + + /** + * 通用的预览失败，导向到不支持的文件响应页面 + * + * @return 页面 + */ + fun notSupportedFile(previewInfo: PreviewInfo, fileType: String, errMsg: String?): PreviewInfo { + previewInfo.msg = htmlEscape(errMsg?:"") + previewInfo.fileTemplate = FilePreview.NOT_SUPPORTED_FILE_PAGE + previewInfo.fileType = htmlEscape(fileType) + return previewInfo + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt new file mode 100644 index 0000000000..fa67ed6a67 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt @@ -0,0 +1,197 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.utils + +import com.tencent.bkrepo.common.api.exception.SystemErrorException +import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.preview.config.configuration.PreviewConfig +import com.tencent.bkrepo.preview.pojo.DownloadResult +import com.tencent.bkrepo.preview.pojo.FileAttribute +import org.apache.http.impl.client.DefaultRedirectStrategy +import org.apache.http.impl.client.HttpClientBuilder +import org.springframework.http.HttpMethod +import org.springframework.http.MediaType +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory +import org.springframework.web.client.RequestCallback +import org.springframework.web.client.RestTemplate +import java.io.File +import java.io.FileNotFoundException +import java.io.IOException +import java.util.* + +/** + * 文件下载工具 + */ +object DownloadUtils { + private val logger = org.slf4j.LoggerFactory.getLogger(DownloadUtils::class.java) + private const val URL_PARAM_FTP_USERNAME = "ftp.username" + private const val URL_PARAM_FTP_PASSWORD = "ftp.password" + private const val URL_PARAM_FTP_CONTROL_ENCODING = "ftp.control.encoding" + private val restTemplate = RestTemplate() + private val factory = HttpComponentsClientHttpRequestFactory() + + /** + * 下载文件 + * @param fileAttribute 文件属性 + * @param config 配置信息 + * @return 下载结果 + */ + @Throws(Exception::class) + fun downLoad(fileAttribute: FileAttribute, config: PreviewConfig): DownloadResult { + var urlStr: String? = null + try { + SslUtils.ignoreSsl() + urlStr = fileAttribute.url?.replace("+", "%20")?.replace(" ", "%20") + } catch (e: Exception) { + logger.error("Ignore SSL certificate exceptions", e) + } + + val result = DownloadResult() + val fileName = fileAttribute.fileName + val realPath = getRelFilePath(fileName, fileAttribute.suffix!!, config.fileDir!!) + + // 判断文件是否合法 + if (FileUtils.isIllegalFileName(realPath)) { + result.apply { + code = DownloadResult.CODE_FAIL + msg = "Download failed,The file name is invalid,$fileName" + } + return result + } + + if (!FileUtils.isAllowedUpload(realPath, config.prohibitSuffix)) { + result.apply { + code = DownloadResult.CODE_FAIL + msg = "Download Failed,Unsupported Type,$urlStr" + } + return result + } + + try { + val url = WebUtils.normalizedURL(urlStr) + if (FileUtils.isHttpUrl(url)) { + val realFile = File(realPath) + factory.setReadTimeout(72000) + factory.setConnectTimeout(10000) + factory.setConnectionRequestTimeout(2000) + factory.httpClient = HttpClientBuilder.create() + .setRedirectStrategy(DefaultRedirectStrategy()) + .build() + restTemplate.requestFactory = factory + val requestCallback = RequestCallback { request -> + request.headers.setAccept(listOf(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL)) + } + + try { + restTemplate.execute(url.toURI(), HttpMethod.GET, requestCallback) { fileResponse -> + org.apache.commons.io.FileUtils.copyToFile(fileResponse.body, realFile) + null + } + } catch (e: Exception) { + logger.error("The download failed: $e") + result.apply { + code = DownloadResult.CODE_FAIL + msg = "The download failed: $e" + } + return result + } + } else if (FileUtils.isFtpUrl(url)) { + val ftpUsername = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_USERNAME) + val ftpPassword = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_PASSWORD) + val ftpControlEncoding = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_CONTROL_ENCODING) + FtpUtils.download( + fileAttribute.url, + realPath, + ftpUsername.takeUnless { it.isNullOrEmpty() } ?: config.ftpUsername, + ftpPassword.takeUnless { it.isNullOrEmpty() } ?: config.ftpPassword, + ftpControlEncoding.takeUnless { it.isNullOrEmpty() } ?: config.ftpControlEncoding + ) + } else { + result.apply { + code = DownloadResult.CODE_FAIL + msg = "Unrecognized URL: $urlStr" + } + } + + FileUtils.getFileMd5AndSize(realPath)?.let { (md5, size) -> + result.apply { + code = DownloadResult.CODE_SUCCESS + filePath = realPath + this.md5 = md5 + this.size = size + msg = fileName + } + } ?: run { + result.apply { + code = DownloadResult.CODE_FAIL + msg = "Failed to calculate MD5 or size for file: $fileName" + } + } + return result + } catch (e: IOException) { + logger.error("File download failed，url：$urlStr") + result.apply { + code = DownloadResult.CODE_FAIL + msg = when (e) { + is FileNotFoundException -> "The file doesn't exist" + else -> e.message ?: "File download failed" + } + } + return result + } + } + + /** + * 文件存放的真实路径 + * @param fileName 文件名 + * @param suffix 文件后缀 + * @param fileDir 根路径 + * @return 文件路径 + */ + fun getRelFilePath(fileName: String?, suffix: String, fileDir: String): String { + var updatedFileName = fileName ?: UUID.randomUUID().toString() + "." + suffix + updatedFileName = updatedFileName.replace( + updatedFileName.substring(updatedFileName.lastIndexOf(".") + 1), + suffix + ) + + val dir = "$fileDir${File.separator}download" + val realPath = "$dir${File.separator}$updatedFileName" + val dirFile = File(dir) + + if (!dirFile.exists() && !dirFile.mkdirs()) { + logger.error("Failed to create directory,$dir") + throw SystemErrorException(CommonMessageCode.SYSTEM_ERROR, "Failed to create directory") + } + return realPath + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/EncodingDetects.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/EncodingDetects.kt new file mode 100644 index 0000000000..54b991cb80 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/EncodingDetects.kt @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.utils + +import org.mozilla.universalchardet.UniversalDetector +import org.slf4j.LoggerFactory +import java.io.File +import java.io.IOException +import java.nio.charset.Charset +import java.nio.file.Files + +/** + * 自动获取文件的编码 + */ +object EncodingDetects { + private const val DEFAULT_LENGTH = 4096 + private const val LIMIT = 50 + private val logger = LoggerFactory.getLogger(EncodingDetects::class.java) + fun getJavaEncode(filePath: String?): String? { + return getJavaEncode(File(filePath)) + } + + fun getJavaEncode(file: File): String? { + val len = Math.min(DEFAULT_LENGTH, file.length().toInt()) + val content = ByteArray(len) + try { + Files.newInputStream(file.toPath()).use { fis -> fis.read(content, 0, len) } + } catch (e: IOException) { + logger.error("File read failed:{}", file.path) + } + return getJavaEncode(content) + } + + fun getJavaEncode(content: ByteArray?): String? { + if (content != null && content.size <= LIMIT) { + return SimpleEncodingDetects.getJavaEncode(content) + } + val detector = UniversalDetector(null) + detector.handleData(content, 0, content!!.size) + detector.dataEnd() + var charsetName: String? = detector.getDetectedCharset() + if (charsetName == null) { + charsetName = Charset.defaultCharset().name() + } + return charsetName + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FileUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FileUtils.kt new file mode 100644 index 0000000000..096a109f0d --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FileUtils.kt @@ -0,0 +1,250 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.utils + +import org.apache.commons.codec.digest.DigestUtils +import org.slf4j.LoggerFactory +import org.springframework.util.ObjectUtils +import org.springframework.util.StringUtils +import org.springframework.web.util.HtmlUtils +import java.io.File +import java.net.URL +import java.util.* +import kotlin.math.log + +object FileUtils { + private val logger = LoggerFactory.getLogger(FileUtils::class.java) + const val DEFAULT_FILE_ENCODING = "UTF-8" + private val illegalFileStrList: MutableList = ArrayList() + + /** + * 检查文件名是否合规 + * + * @param fileName 文件名 + * @return 合规结果, true:不合规，false:合规 + */ + fun isIllegalFileName(fileName: String): Boolean { + for (str in illegalFileStrList) { + if (fileName.contains(str)) { + return true + } + } + return false + } + + /** + * 检查是否是数字 + * + * @param str 文件名 + * @return 合规结果, true:不合规，false:合规 + */ + fun isInteger(str: String?): Boolean { + return !str.isNullOrBlank() && str.matches("-?[0-9]+\\.?[0-9]*".toRegex()) + } + + + /** + * 判断url是否是http资源 + * + * @param url url + * @return 是否http + */ + fun isHttpUrl(url: URL): Boolean { + return url.protocol.lowercase(Locale.getDefault()).startsWith("file") + || url.protocol.lowercase(Locale.getDefault()).startsWith("http") + } + + /** + * 判断url是否是ftp资源 + * + * @param url url + * @return 是否ftp + */ + fun isFtpUrl(url: URL): Boolean { + return "ftp".equals(url.protocol, ignoreCase = true) + } + + /** + * 删除单个文件 + * + * @param fileName 要删除的文件的文件名 + * @return 单个文件删除成功返回true，否则返回false + */ + fun deleteFileByName(fileName: String): Boolean { + val file = File(fileName) + // 如果文件路径所对应的文件存在，并且是一个文件，则直接删除 + return if (file.exists() && file.isFile) { + if (file.delete()) { + logger.debug("Delete a single file [$fileName] success.") + true + } else { + logger.warn("Delete a single file [$fileName] fail.") + false + } + } else { + logger.info("Delete a single file [$fileName] does not exist.") + false + } + } + + fun htmlEscape(input: String): String { + if (StringUtils.hasText(input)) { + //input = input.replaceAll("\\{", "%7B").replaceAll("}", "%7D").replaceAll("\\\\", "%5C"); + val htmlStr = HtmlUtils.htmlEscape(input, "UTF-8") + //& -> & + return htmlStr.replace("&", "&") + } + return input + } + + /** + * 通过文件名获取文件后缀 + * + * @param fileName 文件名称 + * @return 文件后缀 + */ + fun suffixFromFileName(fileName: String): String { + return fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase() + } + + /** + * 根据文件路径删除文件 + * + * @param filePath 绝对路径 + */ + fun deleteFileByPath(filePath: String?) { + val file = File(filePath) + if (file.exists() && !file.delete()) { + logger.warn("File [$filePath] delete failed.") + } + } + + /** + * 删除目录及目录下的文件 + * + * @param dir 要删除的目录的文件路径 + * @return 目录删除成功返回true，否则返回false + */ + fun deleteDirectory(dir: String): Boolean { + var dirPath = dir + // 如果 dir 不以文件分隔符结尾，自动添加文件分隔符 + if (!dirPath.endsWith(File.separator)) { + dirPath += File.separator + } + val dirFile = File(dirPath) + + // 如果 dir 对应的文件不存在，或者不是一个目录，则退出 + if (!dirFile.exists() || !dirFile.isDirectory) { + logger.warn("Failed to delete directory,[$dirPath] does not exist.") + return false + } + + var flag = true + // 删除文件夹中的所有文件，包括子目录 + val files = dirFile.listFiles() ?: return false + for (file in files) { + if (file.isFile) { + flag = deleteFileByName(file.absolutePath) // 自定义的删除文件方法 + if (!flag) { + break + } + } else if (file.isDirectory) { + // 删除子目录 + flag = deleteDirectory(file.absolutePath) + if (!flag) { + break + } + } + } + + // 删除空目录 + if (!dirFile.delete() || !flag) { + logger.warn("Failed to delete directory.") + return false + } + return true + } + + /** + * 判断文件是否允许上传 + * + * @param file 文件扩展名 + * @return 是否允许上传 + */ + fun isAllowedUpload(file: String, prohibit: String): Boolean { + val fileType = suffixFromFileName(file) + for (type in prohibit.split(",".toRegex()).toTypedArray()) { + if (type == fileType) { + return false + } + } + return !ObjectUtils.isEmpty(fileType) + } + + /** + * 判断文件是否存在 + * + * @param filePath 文件路径 + * @return 是否存在 true:存在，false:不存在 + */ + fun isExist(filePath: String?): Boolean { + val file = File(filePath) + return file.exists() + } + + /** + * 获取文件的m5d和大小 + * + * @param filePath 文件绝对路径 + * @return m5d和大小 + */ + fun getFileMd5AndSize(filePath: String): Pair? { + val file = File(filePath) + if (!(file.exists() && file.isFile)) { + logger.warn("Invalid file path: $filePath") + return null + } + val md5Hex = file.inputStream().use { DigestUtils.md5Hex(it) } + return md5Hex to file.length() + } + + init { + illegalFileStrList.add("../") + illegalFileStrList.add("./") + illegalFileStrList.add("..\\") + illegalFileStrList.add(".\\") + illegalFileStrList.add("\\..") + illegalFileStrList.add("\\.") + illegalFileStrList.add("..") + illegalFileStrList.add("...") + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FtpUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FtpUtils.kt new file mode 100644 index 0000000000..7580e3fb9d --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FtpUtils.kt @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.utils + +import org.apache.commons.lang3.StringUtils +import org.apache.commons.net.ftp.FTPClient +import org.apache.commons.net.ftp.FTPReply +import org.slf4j.LoggerFactory +import java.io.IOException +import java.net.URL +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +/** + * FTP下载工具 + */ +object FtpUtils { + private val logger = LoggerFactory.getLogger(FtpUtils::class.java) + + @Throws(IOException::class) + fun connect(host: String?, port: Int, username: String?, password: String?, controlEncoding: String?): FTPClient { + val ftpClient = FTPClient() + ftpClient.connect(host, port) + if (!StringUtils.isEmpty(username) && !StringUtils.isEmpty(password)) { + ftpClient.login(username, password) + } + val reply: Int = ftpClient.getReplyCode() + if (!FTPReply.isPositiveCompletion(reply)) { + ftpClient.disconnect() + } + ftpClient.setControlEncoding(controlEncoding) + ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE) + return ftpClient + } + + @Throws(IOException::class) + fun download( + ftpUrl: String?, + localFilePath: String?, + ftpUsername: String?, + ftpPassword: String?, + ftpControlEncoding: String? + ) { + val url = URL(ftpUrl) + val host = url.host + val port = if (url.port == -1) url.defaultPort else url.port + val remoteFilePath = url.path + logger.debug( + "FTP connection url:{}, username:{}, password:***, controlEncoding:{}, localFilePath:{}", + ftpUrl, + ftpUsername, + ftpControlEncoding, + localFilePath + ) + val ftpClient: FTPClient = connect(host, port, ftpUsername, ftpPassword, ftpControlEncoding) + val outputStream = Files.newOutputStream(Paths.get(localFilePath)) + ftpClient.enterLocalPassiveMode() + val downloadResult: Boolean = ftpClient.retrieveFile( + String( + remoteFilePath.toByteArray( + charset( + ftpControlEncoding!! + ) + ), StandardCharsets.ISO_8859_1 + ), outputStream + ) + logger.debug("FTP download result {}", downloadResult) + outputStream.flush() + outputStream.close() + ftpClient.logout() + ftpClient.disconnect() + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/JsonMapper.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/JsonMapper.kt new file mode 100644 index 0000000000..5b535517d5 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/JsonMapper.kt @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.utils + +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializationFeature +import org.slf4j.LoggerFactory +import java.io.IOException +import java.text.SimpleDateFormat + +/** + * 简单封装Jackson，定制不同的输出 + */ +class JsonMapper @JvmOverloads constructor(include: Include? = null) { + + private val mapper: ObjectMapper = ObjectMapper() + + init { + // 设置输出时包含属性的风格 + include?.let { mapper.setSerializationInclusion(it) } + // 设置输入时忽略在JSON字符串中存在但Java对象实际没有的属性 + mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + mapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + mapper.dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + } + + companion object { + private val logger = LoggerFactory.getLogger(JsonMapper::class.java) + + // 非空属性的序列化 + fun nonEmptyMapper(): JsonMapper = JsonMapper(Include.NON_EMPTY) + + // 非默认值属性的序列化 + fun nonDefaultMapper(): JsonMapper = JsonMapper(Include.NON_DEFAULT) + + // 全部属性序列化 + fun getAllOutputMapper(): JsonMapper = JsonMapper(Include.ALWAYS) + } + + /** + * 将对象转换为 JSON 字符串 + */ + fun toJson(obj: Any?): String? { + return try { + mapper.writeValueAsString(obj) + } catch (e: IOException) { + logger.warn("object to json string error.", e); + null + } + } + + /** + * 反序列化 POJO 或简单集合，如 List。 + * 如果 JSON 字符串为 Null 或 "null" 字符串，返回 Null。 + * 如果 JSON 字符串为 "[]"，返回空集合。 + * 如需反序列化复杂集合，如 List，请使用 fromJson(String, TypeReference)。 + */ + fun fromJson(jsonString: String?, clazz: Class): T? { + if (jsonString.isNullOrEmpty()) return null + + return try { + mapper.readValue(jsonString, clazz) + } catch (e: IOException) { + logger.warn("Parse json string error.", e) + null + } + } + + /** + * 反序列化复杂的泛型对象 + * 如果 JSON 字符串为 Null 或 "null" 字符串，返回 Null。 + * 如果 JSON 字符串为 "[]"，返回空集合。 + */ + fun fromJson(jsonString: String?, typeReference: TypeReference): T? { + if (jsonString.isNullOrEmpty()) return null + + return try { + mapper.readValue(jsonString, typeReference) + } catch (e: IOException) { + logger.warn("Parse json string error.", e) + null + } + } + + /** + * 当 JSON 里只含有 Bean 的部分属性时，更新一个已存在 Bean，只覆盖该部分的属性。 + */ + fun update(jsonString: String?, obj: Any) { + try { + mapper.readerForUpdating(obj).readValue(jsonString) + } catch (e: IOException) { + logger.warn("Update json string to object error.", e) + } + } + + /** + * 获取底层的 ObjectMapper，用于进一步的设置或使用其他序列化 API。 + */ + fun getMapper(): ObjectMapper = mapper +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt new file mode 100644 index 0000000000..e5b27c27b4 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.utils + +import java.security.cert.X509Certificate +import javax.net.ssl.* + +/** + * SSL工具 + */ +object SslUtils { + + /** + * 忽略所有HTTPS请求的SSL证书 + * 必须在openConnection之前调用 + */ + @Throws(Exception::class) + fun ignoreSsl() { + val hv = HostnameVerifier { _, _ -> true } + trustAllHttpsCertificates() + HttpsURLConnection.setDefaultHostnameVerifier(hv) + } + + /** + * 信任所有HTTPS证书 + */ + @Throws(Exception::class) + private fun trustAllHttpsCertificates() { + val trustAllCerts = arrayOf(miTM()) + val sc = SSLContext.getInstance("SSL") + sc.init(null, trustAllCerts, null) + HttpsURLConnection.setDefaultSSLSocketFactory(sc.socketFactory) + } + + // 自定义 TrustManager 实现 + private class miTM : X509TrustManager { + override fun getAcceptedIssuers(): Array? = null + + override fun checkServerTrusted(certs: Array, authType: String) {} + + override fun checkClientTrusted(certs: Array, authType: String) {} + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/UrlEncoderUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/UrlEncoderUtils.kt new file mode 100644 index 0000000000..47f29fdf86 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/UrlEncoderUtils.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.utils + +import java.util.* + +/** + * url编码工具 + */ +object UrlEncoderUtils { + + // BitSet is initialized with 256 size to track characters that do not need encoding + private val dontNeedEncoding = BitSet(256) + + init { + // Define the characters that do not need encoding + for (i in 'a'..'z') dontNeedEncoding.set(i.code) + for (i in 'A'..'Z') dontNeedEncoding.set(i.code) + for (i in '0'..'9') dontNeedEncoding.set(i.code) + dontNeedEncoding.set('+'.code) // '+' is considered safe + dontNeedEncoding.set('-'.code) + dontNeedEncoding.set('_'.code) + dontNeedEncoding.set('.'.code) + dontNeedEncoding.set('*'.code) + } + + /** + * 判断str是否urlEncoder.encode过 + * 经常遇到这样的情况，拿到一个URL,但是搞不清楚到底要不要encode. + * 不做encode吧，担心出错，做encode吧，又怕重复了 + * + * @param str 输入的字符串 + * @return 是否已经进行过URL编码 + */ + fun hasUrlEncoded(str: String): Boolean { + var needEncode = false + + for (i in str.indices) { + val c = str[i] + if (dontNeedEncoding[c.toInt()]) { + continue + } + if (c == '%' && i + 2 < str.length) { + // 判断是否符合urlEncode规范 + val c1 = str[i + 1] + val c2 = str[i + 2] + if (isDigit16Char(c1) && isDigit16Char(c2)) { + continue + } + } + // 其他字符，肯定需要urlEncode + needEncode = true + break + } + + return !needEncode + } + + /** + * 判断字符是否是16进制的字符 + * + * @param c 字符 + * @return 是否是16进制字符 + */ + private fun isDigit16Char(c: Char): Boolean { + return c in '0'..'9' || c in 'A'..'F' + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/WebUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/WebUtils.kt new file mode 100644 index 0000000000..950c30c7e5 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/WebUtils.kt @@ -0,0 +1,369 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview.utils + +import io.mola.galimatias.GalimatiasParseException +import org.slf4j.LoggerFactory +import org.springframework.util.Base64Utils +import org.springframework.util.StringUtils +import org.springframework.web.multipart.MultipartFile +import org.springframework.web.util.HtmlUtils +import java.io.UnsupportedEncodingException +import java.net.MalformedURLException +import java.net.URL +import java.net.URLEncoder +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets +import java.util.* +import java.util.regex.Pattern +import javax.servlet.ServletRequest +import javax.servlet.http.HttpServletRequest + +object WebUtils { + private val logger = LoggerFactory.getLogger(WebUtils::class.java) + + /** + * 获取标准的URL + * + * @param urlStr url + * @return 标准的URL + */ + @Throws(GalimatiasParseException::class, MalformedURLException::class) + fun normalizedURL(urlStr: String?): URL { + return io.mola.galimatias.URL.parse(urlStr).toJavaURL() + } + + /** + * 对文件名进行编码 + * + */ + fun encodeFileName(name: String?): String? { + var name = name + name = try { + URLEncoder.encode(name, "UTF-8").replace("\\+".toRegex(), "%20") + } catch (e: UnsupportedEncodingException) { + return null + } + return name + } + + /** + * 去除fullfilename参数 + * + * @param urlStr + * @return + */ + fun clearFullfilenameParam(urlStr: String?): String { + // 去除特定参数字段 + val pattern = Pattern.compile("(&fullfilename=[^&]*)") + val matcher = pattern.matcher(urlStr) + return matcher.replaceAll("") + } + + /** + * 对URL进行编码 + */ + fun urlEncoderencode(urlStr: String): String { + var urlStr = urlStr + var fullFileName = getUrlParameterReg(urlStr, "fullfilename") //获取流文件名 + if (StringUtils.hasText(fullFileName)) { + // 移除fullfilename参数 + urlStr = clearFullfilenameParam(urlStr) + } else { + fullFileName = getFileNameFromURL(urlStr) //获取文件名 + } + if (!fullFileName?.let { UrlEncoderUtils.hasUrlEncoded(it) }!!) { //判断文件名是否转义 + try { + urlStr = + URLEncoder.encode(urlStr, "UTF-8").replace("\\+".toRegex(), "%20") + .replace("%3A".toRegex(), ":") + .replace("%2F".toRegex(), "/") + .replace("%3F".toRegex(), "?") + .replace("%26".toRegex(), "&") + .replace("%3D".toRegex(), "=") + } catch (e: UnsupportedEncodingException) { + e.printStackTrace() + } + } + return urlStr + } + + /** + * 获取url中的参数 + * + * @param url url + * @param name 参数名 + * @return 参数值 + */ + fun getUrlParameterReg(url: String, name: String): String? { + val mapRequest: MutableMap = HashMap() + val strUrlParam = truncateUrlPage(url) ?: return "" + //每个键值为一组 + val arrSplit = strUrlParam.split("[&]".toRegex()).toTypedArray() + for (strSplit in arrSplit) { + val arrSplitEqual = strSplit.split("[=]".toRegex()).toTypedArray() + //解析出键值 + if (arrSplitEqual.size > 1) { + //正确解析 + mapRequest[arrSplitEqual[0]] = arrSplitEqual[1] + } else if (arrSplitEqual[0] != "") { + //只有参数没有值，不加入 + mapRequest[arrSplitEqual[0]] = "" + } + } + return mapRequest[name] + } + + /** + * 去掉url中的路径，留下请求参数部分 + * + * @param strURL url地址 + * @return url请求参数部分 + */ + private fun truncateUrlPage(strURL: String): String? { + var strURL = strURL + var strAllParam: String? = null + strURL = strURL.trim { it <= ' ' } + val arrSplit: Array = strURL.split("[?]".toRegex()).toTypedArray() + if (strURL.length > 1) { + if (arrSplit.size > 1) { + if (arrSplit[1] != null) { + strAllParam = arrSplit[1] + } + } + } + return strAllParam + } + + /** + * 从url中剥离出文件名 + * + * @param url + * @return 文件名 + */ + fun getFileNameFromURL(url: String): String { + var url = url + if (url.toLowerCase().startsWith("file:")) { + try { + val urlObj = URL(url) + url = urlObj.path.substring(1) + } catch (e: MalformedURLException) { + e.printStackTrace() + } + } + // 因为url的参数中可能会存在/的情况，所以直接url.lastIndexOf("/")会有问题 + // 所以先从？处将url截断，然后运用url.lastIndexOf("/")获取文件名 + val noQueryUrl = url.substring(0, if (url.contains("?")) url.indexOf("?") else url.length) + return noQueryUrl.substring(noQueryUrl.lastIndexOf("/") + 1) + } + + /** + * 从url中剥离出文件名 + * @param file 文件 + * @return 文件名 + */ + fun getFileNameFromMultipartFile(file: MultipartFile): String? { + var fileName = file.originalFilename!! + fileName = HtmlUtils.htmlEscape(fileName, FileUtils.DEFAULT_FILE_ENCODING) + + // Check for Unix-style path + val unixSep = fileName.lastIndexOf('/') + // Check for Windows-style path + val winSep = fileName.lastIndexOf('\\') + // Cut off at latest possible point + val pos = Math.max(winSep, unixSep) + if (pos != -1) { + fileName = fileName.substring(pos + 1) + } + return fileName + } + + /** + * 从url中获取文件后缀 + * + * @param url url + * @return 文件后缀 + */ + fun suffixFromUrl(url: String): String { + val nonPramStr = url.substring(0, if (url.contains("?")) url.indexOf("?") else url.length) + val fileName = nonPramStr.substring(nonPramStr.lastIndexOf("/") + 1) + return FileUtils.suffixFromFileName(fileName) + } + + /** + * 对url中的文件名进行UTF-8编码 + * + * @param url url + * @return 文件名编码后的url + */ + fun encodeUrlFileName(url: String): String? { + val encodedFileName: String + val noQueryUrl = url.substring(0, if (url.contains("?")) url.indexOf("?") else url.length) + val fileNameStartIndex = noQueryUrl.lastIndexOf('/') + 1 + val fileNameEndIndex = noQueryUrl.lastIndexOf('.') + if (fileNameEndIndex < fileNameStartIndex) { + return url + } + encodedFileName = try { + URLEncoder.encode(noQueryUrl.substring(fileNameStartIndex, fileNameEndIndex), "UTF-8") + } catch (e: UnsupportedEncodingException) { + return null + } + return url.substring(0, fileNameStartIndex) + encodedFileName + url.substring(fileNameEndIndex) + } + + /** + * 从 ServletRequest 获取预览的源 url , 已 base64 解码 + * + * @param request 请求 request + * @return url + */ + fun getSourceUrl(request: ServletRequest): String? { + val url = request.getParameter("url") + var urls = request.getParameter("urls") + val currentUrl = request.getParameter("currentUrl") + val urlPath = request.getParameter("urlPath") + if (org.apache.commons.lang3.StringUtils.isNotBlank(url)) { + return decodeUrl(url) + } + if (org.apache.commons.lang3.StringUtils.isNotBlank(currentUrl)) { + return decodeUrl(currentUrl) + } + if (org.apache.commons.lang3.StringUtils.isNotBlank(urlPath)) { + return decodeUrl(urlPath) + } + if (org.apache.commons.lang3.StringUtils.isNotBlank(urls)) { + urls = decodeUrl(urls) + val images = urls!!.split("\\|".toRegex()).toTypedArray() + return images[0] + } + return null + } + + /** + * 判断地址是否正确 + * 高 2022/12/17 + */ + fun isValidUrl(url: String?): Boolean { + val regStr = "^((https|http|ftp|rtsp|mms|file)://)" //[.?*]表示匹配的就是本身 + val pattern = Pattern.compile(regStr) + val matcher = pattern.matcher(url) + return matcher.find() + } + + /** + * 将 Base64 字符串解码，再解码URL参数, 默认使用 UTF-8 + * @param source 原始 Base64 字符串 + * @return decoded string + * + */ + fun decodeUrl(source: String): String? { + val url = decodeBase64String(source, StandardCharsets.UTF_8) + return if (!org.apache.commons.lang3.StringUtils.isNotBlank(url)) { + null + } else url + } + + /** + * 将 Base64 字符串使用指定字符集解码 + * @param source 原始 Base64 字符串 + * @param charsets 字符集 + * @return decoded string + */ + fun decodeBase64String(source: String?, charsets: Charset?): String? { + /* + * url 传入的参数里加号会被替换成空格，导致解析出错，这里需要把空格替换回加号 + * 有些 Base64 实现可能每 76 个字符插入换行符，也一并去掉 + */ + return String( + Base64Utils.decodeFromString(source!!.replace(" ".toRegex(), "+") + .replace("\n".toRegex(), "")), + charsets!! + ) + } + + /** + * 获取 url 的 host + * @param urlStr url + * @return host + */ + fun getHost(urlStr: String?): String? { + try { + val url = URL(urlStr) + return url.host.toLowerCase() + } catch (ignored: MalformedURLException) { + } + return null + } + + /** + * 获取 session 中的 String 属性 + * @param request 请求 + * @return 属性值 + */ + fun getSessionAttr(request: HttpServletRequest, key: String?): String? { + val session = request.session ?: return null + val value = session.getAttribute(key) ?: return null + return value.toString() + } + + /** + * 获取 session 中的 long 属性 + * @param request 请求 + * @param key 属性名 + * @return 属性值 + */ + fun getLongSessionAttr(request: HttpServletRequest, key: String?): Long { + val value = getSessionAttr(request, key) ?: return 0 + return value.toLong() + } + + /** + * session 中设置属性 + * @param request 请求 + * @param key 属性名 + */ + fun setSessionAttr(request: HttpServletRequest, key: String?, value: Any?) { + val session = request.session ?: return + session.setAttribute(key, value) + } + + /** + * 移除 session 中的属性 + * @param request 请求 + * @param key 属性名 + */ + fun removeSessionAttr(request: HttpServletRequest, key: String?) { + val session = request.session ?: return + session.removeAttribute(key) + } +} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/resources/i18n/messages_en.properties b/src/backend/preview/biz-preview/src/main/resources/i18n/messages_en.properties new file mode 100644 index 0000000000..7213e7b174 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/resources/i18n/messages_en.properties @@ -0,0 +1,36 @@ +# +# Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. +# +# Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +# +# BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. +# +# A copy of the MIT License is included in this file. +# +# +# Terms of the MIT License: +# --------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +preview.file.not-found=file [{0}] not found +preview.node.not-found=node [{0}] not found +preview.repo.not-found=repo [{0}] not found +preview.file.handle.error=file [{0}] convert fail +preview.parameter.invalid=param [{0}] is illegal \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/preview/biz-preview/src/main/resources/i18n/messages_zh_CN.properties new file mode 100644 index 0000000000..b73d382a61 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/resources/i18n/messages_zh_CN.properties @@ -0,0 +1,36 @@ +# +# Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. +# +# Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +# +# BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. +# +# A copy of the MIT License is included in this file. +# +# +# Terms of the MIT License: +# --------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +preview.file.not-found=文件[{0}]不存在 +preview.node.not-found=节点[{0}]不存在 +preview.repo.not-found=仓库[{0}]不存在 +preview.file.handle.error=文件[{0}]转换失败 +preview.parameter.invalid=参数[{0}]不合法 diff --git a/src/backend/preview/biz-preview/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/preview/biz-preview/src/main/resources/i18n/messages_zh_TW.properties new file mode 100644 index 0000000000..a2b153afb1 --- /dev/null +++ b/src/backend/preview/biz-preview/src/main/resources/i18n/messages_zh_TW.properties @@ -0,0 +1,36 @@ +# +# Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. +# +# Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +# +# BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. +# +# A copy of the MIT License is included in this file. +# +# +# Terms of the MIT License: +# --------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +preview.file.not-found=文件[{0}]不存在 +preview.node.not-found=節點[{0}]不存在 +preview.repo.not-found=倉庫[{0}]不存在 +preview.file.handle.error=文件[{0}]轉換失敗 +preview.parameter.invalid=參數[{0}]不合法 diff --git a/src/backend/preview/boot-preview/build.gradle b/src/backend/preview/boot-preview/build.gradle new file mode 100644 index 0000000000..2717d0dba7 --- /dev/null +++ b/src/backend/preview/boot-preview/build.gradle @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +dependencies { + implementation(project(":preview:biz-preview")) +} diff --git a/src/backend/preview/boot-preview/src/main/kotlin/com/tencent/bkrepo/preview/PreviewApplication.kt b/src/backend/preview/boot-preview/src/main/kotlin/com/tencent/bkrepo/preview/PreviewApplication.kt new file mode 100644 index 0000000000..2f49638d3a --- /dev/null +++ b/src/backend/preview/boot-preview/src/main/kotlin/com/tencent/bkrepo/preview/PreviewApplication.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.preview + +import com.tencent.bkrepo.common.service.condition.MicroService +import org.springframework.boot.runApplication + +/** + * 文件预览微服务启动类 + */ +@MicroService +class PreviewApplication + +fun main(args: Array) { + runApplication(*args) +} \ No newline at end of file diff --git a/src/backend/preview/boot-preview/src/main/resources/bootstrap.yml b/src/backend/preview/boot-preview/src/main/resources/bootstrap.yml new file mode 100644 index 0000000000..10b07d2160 --- /dev/null +++ b/src/backend/preview/boot-preview/src/main/resources/bootstrap.yml @@ -0,0 +1,2 @@ +spring.application.name: preview +server.port: 25822 \ No newline at end of file diff --git a/src/backend/preview/build.gradle b/src/backend/preview/build.gradle new file mode 100644 index 0000000000..27b9957954 --- /dev/null +++ b/src/backend/preview/build.gradle @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ \ No newline at end of file diff --git a/src/backend/settings.gradle.kts b/src/backend/settings.gradle.kts index 23128e5dcc..74159e543f 100644 --- a/src/backend/settings.gradle.kts +++ b/src/backend/settings.gradle.kts @@ -93,3 +93,4 @@ includeAll(":router-controller") includeAll(":media") includeAll(":common:common-metadata") includeAll(":common:common-service") +includeAll(":preview") diff --git a/support-files/kubernetes/charts/bkrepo/templates/preview/configmap.yaml b/support-files/kubernetes/charts/bkrepo/templates/preview/configmap.yaml new file mode 100644 index 0000000000..3962bdfbb8 --- /dev/null +++ b/support-files/kubernetes/charts/bkrepo/templates/preview/configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.preview.enabled -}} +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ include "common.names.fullname" . }}-preview + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: preview + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +data: + application.yml: |- + preview: + domain: {{ .Values.bkDomainScheme }}://{{ .Values.gateway.host }}/preview + genericDomain: {{ .Values.bkDomainScheme }}://{{ .Values.gateway.host }}/generic + {{- if keys $.Values.preview.config }} + {{- toYaml .Values.preview.config | nindent 6 }} + {{- end}} +{{- end }} \ No newline at end of file diff --git a/support-files/kubernetes/charts/bkrepo/templates/preview/deployment.yaml b/support-files/kubernetes/charts/bkrepo/templates/preview/deployment.yaml new file mode 100644 index 0000000000..2086ab77c3 --- /dev/null +++ b/support-files/kubernetes/charts/bkrepo/templates/preview/deployment.yaml @@ -0,0 +1,137 @@ +{{- if .Values.preview.enabled -}} +apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }}-preview + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: preview + {{ include "bkrepo.labelValues.scope" . }}: {{ include "bkrepo.labelValues.scope.backend" . }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: preview + replicas: {{ default 1 .Values.preview.replicaCount }} + template: + metadata: + labels: {{- include "common.labels.standard" . | nindent 8 }} + app.kubernetes.io/component: preview + {{ include "bkrepo.labelValues.scope" . }}: {{ include "bkrepo.labelValues.scope.backend" . }} + {{- if .Values.preview.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.preview.podLabels "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "bkrepo.serviceAccountName" . }} + {{- include "bkrepo.imagePullSecrets" . | nindent 6 }} + {{- if .Values.preview.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.preview.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.preview.affinity }} + affinity: {{- include "common.tplvalues.render" ( dict "value" .Values.preview.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + {{- if eq .Values.persistence.accessMode "ReadWriteOnce" }} + podAffinity: {{- include "common.affinities.pods" (dict "type" "soft" "component" "repository" "context" $) | nindent 10 }} + {{- else }} + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.repository.podAffinityPreset "component" "preview" "context" $) | nindent 10 }} + {{- end }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.preview.podAntiAffinityPreset "component" "preview" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.preview.nodeAffinityPreset.type "key" .Values.preview.nodeAffinityPreset.key "values" .Values.preview.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} + {{- if .Values.preview.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" ( dict "value" .Values.preview.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.preview.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.preview.tolerations "context" .) | nindent 8 }} + {{- end }} + {{- if .Values.preview.priorityClassName }} + priorityClassName: {{ .Values.preview.priorityClassName | quote }} + {{- end }} + {{- if .Values.preview.podSecurityContext.enabled }} + securityContext: {{- omit .Values.preview.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + containers: + {{- if .Values.bkstore.enabled }} + - name: plugin-bkstore + image: {{ include "bkrepo.images.image" ( dict "imageRoot" .Values.bkstore.image "global" .Values.global "bkrepo" .Values.common) }} + args: + - /bin/sh + - -c + - cp /data/workspace/plugin*jar /data/workspace/plugin/; sleep infinity + volumeMounts: + - name: shared-data + mountPath: /data/workspace/plugin/ + {{- end }} + {{- if .Values.gitci.enabled }} + - name: plugin-gitci + image: {{ include "bkrepo.images.image" ( dict "imageRoot" .Values.gitci.image "global" .Values.global "bkrepo" .Values.common) }} + args: + - /bin/sh + - -c + - cp /data/workspace/plugin*jar /data/workspace/plugin/; sleep infinity + volumeMounts: + - name: shared-data + mountPath: /data/workspace/plugin/ + {{- end }} + - name: preview + image: {{ include "bkrepo.images.image" ( dict "imageRoot" .Values.preview.image "global" .Values.global "bkrepo" .Values.common) }} + imagePullPolicy: {{ .Values.preview.image.pullPolicy }} + {{- if .Values.preview.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.preview.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.preview.resources }} + resources: {{- toYaml .Values.preview.resources | nindent 12 }} + {{- end }} + env: + - name: BK_REPO_JVM_OPTION + value: {{ .Values.common.jvmOption }} + - name: BK_REPO_PROFILE + value: {{ .Values.common.springProfile }} + - name: BK_REPO_SERVICE_PREFIX + value: {{ include "common.names.fullname" . }}- + ports: + - name: http + containerPort: 25801 + protocol: TCP + livenessProbe: + httpGet: + path: /actuator/health/livenessState + port: http + initialDelaySeconds: 120 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 1 + readinessProbe: + httpGet: + path: /actuator/health/readinessState + port: http + initialDelaySeconds: 60 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 1 + volumeMounts: + - name: storage + mountPath: {{ .Values.common.mountPath }} + - name: shared-data + mountPath: /data/workspace/plugin/ + volumes: + - name: storage + {{- if .Values.common.config.storage.nfs.enabled }} + persistentVolumeClaim: + claimName: {{ include "common.names.fullname" . }}-nfs-pvc + {{- else if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "common.names.fullname" . }}-storage{{- end }} + {{- else }} + emptyDir: {} + {{- end }} + - name: shared-data + emptyDir: {} +{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/templates/preview/service.yaml b/support-files/kubernetes/charts/bkrepo/templates/preview/service.yaml new file mode 100644 index 0000000000..4253539622 --- /dev/null +++ b/support-files/kubernetes/charts/bkrepo/templates/preview/service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.preview.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.names.fullname" . }}-preview + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: preview + {{ include "bkrepo.labelValues.scope" . }}: {{ include "bkrepo.labelValues.scope.backend" . }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + selector: {{- include "common.labels.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: preview +{{- end }} diff --git a/support-files/kubernetes/charts/bkrepo/values.yaml b/support-files/kubernetes/charts/bkrepo/values.yaml index fba4c2a10f..b3937f805b 100644 --- a/support-files/kubernetes/charts/bkrepo/values.yaml +++ b/support-files/kubernetes/charts/bkrepo/values.yaml @@ -545,6 +545,46 @@ s3: podAnnotations: {} priorityClassName: "" +## preview服务配置 +preview: + enabled: true + config: {} + ## Kubernetes 通用配置 + image: + registry: registry.hub.docker.com + repository: bkrepo/bkrepo-preview + tag: 1.1.0 + pullPolicy: IfNotPresent + pullSecrets: [] + replicaCount: 1 + hostAliases: [] + resources: + requests: + cpu: 100m + memory: 1000Mi + limits: + cpu: 500m + memory: 1500Mi + containerSecurityContext: + enabled: false + runAsUser: 1001 + runAsNonRoot: true + podSecurityContext: + enabled: false + fsGroup: 1001 + podAffinityPreset: "" + podAntiAffinityPreset: "" + nodeAffinityPreset: + type: "" + key: "" + values: [] + affinity: {} + nodeSelector: {} + tolerations: [] + podLabels: {} + podAnnotations: {} + priorityClassName: "" + ## maven registry服务配置 maven: enabled: true diff --git a/support-files/kubernetes/images/backend/preview.Dockerfile b/support-files/kubernetes/images/backend/preview.Dockerfile new file mode 100644 index 0000000000..7ce332a8d5 --- /dev/null +++ b/support-files/kubernetes/images/backend/preview.Dockerfile @@ -0,0 +1,47 @@ +FROM blueking/jdk:0.0.2 + +LABEL maintainer="Tencent BlueKing Devops" + +# 内置一些常用的中文字体，避免普遍性乱码 +# COPY fonts/* /usr/share/fonts/chinese/ +RUN yum -y install wqy-zenhei wqy-microhei fontconfig && \ + fc-cache -fv && \ + yum -y install ca-certificates && \ + # 安装 glibc-common 以支持中文本地化 + yum -y install glibc-common && \ + # 安装必要的字体包 + yum -y install wqy-zenhei wqy-microhei ttf-dejavu fontconfig wget && \ + # 设置中文环境 + localedef -i zh_CN -c -f UTF-8 -A /usr/share/locale/locale.alias zh_CN.UTF-8 && \ + export LANG=zh_CN.UTF-8 && \ + # 设置时区 + export DEBIAN_FRONTEND=noninteractive && \ + yum -y install tzdata && \ + ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + # 安装 LibreOffice 所需的库 + yum -y install libXrender libXinerama libXt libXext libfreetype cairo cups libX11 nss && \ + # 清理缓存 + yum clean all && \ + # 下载 LibreOffice RPM 包 + wget https://downloadarchive.documentfoundation.org/libreoffice/old/7.6.7.1/rpm/x86_64/LibreOffice_7.6.7.1_Linux_x86-64_rpm.tar.gz -O libreoffice_rpm.tar.gz && \ + tar -zxf libreoffice_rpm.tar.gz -C /tmp && \ + # 安装 RPM 包 + cd /tmp/LibreOffice_7.6.7.1_Linux_x86-64_rpm/RPMS && \ + yum -y install *.rpm && \ + # 清理临时文件 + rm -rf /tmp/* && rm -rf /var/lib/apt/lists/* + +ENV BK_REPO_HOME=/data/workspace \ + BK_REPO_LOGS_DIR=/data/workspace/logs \ + BK_REPO_SERVICE_PREFIX=bkrepo- \ + BK_REPO_PROFILE=dev + +RUN mkdir -p /data/tools && \ + curl -o /data/tools/arthas.jar https://arthas.aliyun.com/arthas-boot.jar + +COPY ./ /data/workspace/ +RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo 'Asia/Shanghai' > /etc/timezone && \ + chmod +x /data/workspace/startup.sh +WORKDIR /data/workspace +CMD /data/workspace/startup.sh From 68cce24fa995980960abadac2d04e2a78291505a Mon Sep 17 00:00:00 2001 From: liuliaozhong Date: Tue, 10 Dec 2024 11:03:36 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat:=20=E6=96=87=E4=BB=B6=E5=9C=A8?= =?UTF-8?q?=E7=BA=BF=E9=A2=84=E8=A7=88=E6=94=AF=E6=8C=81=E6=9B=B4=E5=A4=9A?= =?UTF-8?q?=E7=9A=84=E6=96=87=E4=BB=B6=E7=B1=BB=E5=9E=8B=20#2765?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 代码规范 --- .../bkrepo/preview/pojo/FileAttribute.kt | 1 - .../config/configuration/PreviewConfig.kt | 2 +- .../controller/FilePreviewController.kt | 6 +- .../preview/service/FileHandlerService.kt | 91 ++------- .../preview/service/FilePreviewFactory.kt | 2 - .../preview/service/FileTransferService.kt | 4 +- .../preview/service/OfficePluginManager.kt | 3 - .../bkrepo/preview/utils/DownloadUtils.kt | 185 +++++++++++------- .../tencent/bkrepo/preview/utils/FileUtils.kt | 3 +- .../tencent/bkrepo/preview/utils/SslUtils.kt | 14 +- 10 files changed, 139 insertions(+), 172 deletions(-) diff --git a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileAttribute.kt b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileAttribute.kt index 1ebb7e1cfa..8f8bf7a005 100644 --- a/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileAttribute.kt +++ b/src/backend/preview/api-preview/src/main/kotlin/com/tencent/bkrepo/preview/pojo/FileAttribute.kt @@ -32,7 +32,6 @@ package com.tencent.bkrepo.preview.pojo import io.swagger.annotations.ApiModel -import io.swagger.annotations.ApiModelProperty @ApiModel("预览文件信息") data class FileAttribute ( diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt index e9cd5e2b62..b39909a503 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt @@ -65,7 +65,7 @@ class PreviewConfig { * 预览仓库配额,默认10G */ @Value("\${preview.repoQuota:10485760}") - val repoQuota: Long? = 10485760 + val repoQuota: Long? = 1073741824 /** * 预览仓库制品保留天数,默认7天 diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/FilePreviewController.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/FilePreviewController.kt index 29f6e1be29..61eedbd76d 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/FilePreviewController.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/controller/FilePreviewController.kt @@ -31,9 +31,7 @@ package com.tencent.bkrepo.preview.controller -import com.tencent.bkrepo.common.api.exception.ParameterInvalidException import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.preview.artifact.PreviewArtifactInfo @@ -41,14 +39,12 @@ import com.tencent.bkrepo.preview.artifact.PreviewArtifactInfo.Companion.PREVIEW import com.tencent.bkrepo.preview.artifact.PreviewArtifactInfo.Companion.PREVIEW_REMOTE_MAPPING_URI import com.tencent.bkrepo.preview.constant.PreviewMessageCode import com.tencent.bkrepo.preview.exception.PreviewInvalidException +import com.tencent.bkrepo.preview.pojo.PreviewInfo import com.tencent.bkrepo.preview.service.FileHandlerService import com.tencent.bkrepo.preview.service.FilePreviewFactory import com.tencent.bkrepo.preview.utils.WebUtils -import com.tencent.bkrepo.preview.pojo.PreviewInfo import org.slf4j.LoggerFactory -import org.springframework.http.HttpStatus import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController import javax.servlet.http.HttpServletRequest diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt index b421bfa441..2ec3c61c8a 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt @@ -1,21 +1,21 @@ package com.tencent.bkrepo.preview.service -import com.tencent.bkrepo.common.api.exception.ParameterInvalidException import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.preview.config.configuration.PreviewConfig import com.tencent.bkrepo.preview.constant.PreviewMessageCode import com.tencent.bkrepo.preview.exception.PreviewInvalidException -import com.tencent.bkrepo.preview.utils.* import com.tencent.bkrepo.preview.pojo.FileAttribute import com.tencent.bkrepo.preview.pojo.FileType +import com.tencent.bkrepo.preview.utils.FileUtils +import com.tencent.bkrepo.preview.utils.JsonMapper +import com.tencent.bkrepo.preview.utils.WebUtils +import com.tencent.bkrepo.preview.utils.UrlEncoderUtils import org.slf4j.LoggerFactory import org.springframework.stereotype.Component -import org.springframework.util.StringUtils -import java.io.* +import java.io.File +import java.io.UnsupportedEncodingException import java.net.URLDecoder -import java.net.URLEncoder -import java.nio.charset.Charset -import java.util.* +import java.util.UUID import javax.servlet.http.HttpServletRequest @Component @@ -27,70 +27,6 @@ class FileHandlerService(private val config: PreviewConfig) { private val logger = LoggerFactory.getLogger(FileHandlerService::class.java) } - /** - * 对转换后的文件进行操作(改变编码方式) - * - * @param outFilePath 文件绝对路径 - */ - fun doActionConvertedFile(outFilePath: String) { - val charset = EncodingDetects.getJavaEncode(outFilePath) // 假设返回的是 String? - val sb = StringBuilder() - try { - FileInputStream(outFilePath).bufferedReader(charset = Charset.forName(charset)).use { reader -> - var line: String? - while (reader.readLine().also { line = it } != null) { - if (line!!.contains("charset=gb2312")) { - line = line!!.replace("charset=gb2312", "charset=utf-8") - } - sb.append(line) - } - // 添加sheet控制头 - sb.append("") - sb.append("") - sb.append("") - } - } catch (e: IOException) { - e.printStackTrace() - } - // 重新写入文件 - try { - FileOutputStream(outFilePath).bufferedWriter(Charset.forName("UTF-8")).use { writer -> - writer.write(sb.toString()) - } - } catch (e: IOException) { - e.printStackTrace() - } - } - - /** - * 获取本地 pdf 转 image 后的 web 访问地址 - * - * @param pdfFilePath pdf文件名 - * @param index 图片索引 - * @return 图片访问地址 - */ - private fun getPdf2jpgUrl(pdfFilePath: String, index: Int): String { - val baseUrl = "" // 设置 baseUrl - val pdfFolder = pdfFilePath.replace(config.fileDir, "").removeSuffix(".pdf") - val urlPrefix: String - urlPrefix = try { - "$baseUrl${URLEncoder.encode(pdfFolder, URI_ENCODING).replace("+", "%20")}" - } catch (e: UnsupportedEncodingException) { - logger.error("UnsupportedEncodingException", e) - "$baseUrl$pdfFolder" - } - return "$urlPrefix/$index$PDF2JPG_IMAGE_FORMAT" - } - - /** - * @param str 原字符串（待截取原串） - * @param posStr 指定字符串 - * @return 截取指定字符串之后的数据 - */ - fun getSubString(str: String, posStr: String): String { - return str.substring(str.indexOf(posStr) + posStr.length) - } - /** * 获取文件属性 * @@ -168,16 +104,17 @@ class FileHandlerService(private val config: PreviewConfig) { return baseUrl } - - /** * 请求校验 */ private fun checkRequest(fileAttribute: FileAttribute, req: HttpServletRequest) { - if (fileAttribute.projectId.isNullOrEmpty() - && fileAttribute.repoName.isNullOrEmpty() - && fileAttribute.artifactUri.isNullOrEmpty() - && fileAttribute.url.isNullOrEmpty()) { + val isInvalid = listOf( + fileAttribute.projectId, + fileAttribute.repoName, + fileAttribute.artifactUri, + fileAttribute.url + ).all { it.isNullOrEmpty() } + if (isInvalid) { logger.error("The file download info cannot be empty") throw PreviewInvalidException(PreviewMessageCode.PREVIEW_PARAMETER_INVALID, "url") } diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreviewFactory.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreviewFactory.kt index c193eeb766..077f19c646 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreviewFactory.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FilePreviewFactory.kt @@ -1,8 +1,6 @@ package com.tencent.bkrepo.preview.service import com.tencent.bkrepo.preview.pojo.FileAttribute -import com.tencent.bkrepo.preview.pojo.FileType -import com.tencent.bkrepo.preview.service.FilePreview import org.springframework.context.ApplicationContext import org.springframework.stereotype.Service diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileTransferService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileTransferService.kt index 69631d0966..4abe207d6f 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileTransferService.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileTransferService.kt @@ -34,8 +34,6 @@ package com.tencent.bkrepo.preview.service import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.constant.ARTIFACT_INFO_KEY import com.tencent.bkrepo.common.artifact.constant.REPO_KEY -import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder @@ -59,7 +57,7 @@ import org.springframework.stereotype.Component import java.io.File import java.time.LocalDateTime import java.time.format.DateTimeFormatter -import java.util.* +import java.util.UUID import javax.servlet.http.HttpServletRequest @Component diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt index 5ca09a68c0..946158e026 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt @@ -33,7 +33,6 @@ package com.tencent.bkrepo.preview.service import org.apache.commons.lang3.StringUtils import org.jodconverter.core.office.InstalledOfficeManagerHolder -import org.jodconverter.core.office.OfficeException import org.jodconverter.core.office.OfficeUtils import org.jodconverter.core.util.OSUtils import org.jodconverter.local.office.LocalOfficeManager @@ -45,8 +44,6 @@ import org.springframework.stereotype.Component import java.io.ByteArrayOutputStream import java.io.File import java.io.IOException -import java.util.* -import java.util.function.ToIntFunction import javax.annotation.PostConstruct import javax.annotation.PreDestroy diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt index fa67ed6a67..1b3bb79647 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt @@ -46,7 +46,8 @@ import org.springframework.web.client.RestTemplate import java.io.File import java.io.FileNotFoundException import java.io.IOException -import java.util.* +import java.net.URL +import java.util.UUID /** * 文件下载工具 @@ -67,74 +68,20 @@ object DownloadUtils { */ @Throws(Exception::class) fun downLoad(fileAttribute: FileAttribute, config: PreviewConfig): DownloadResult { - var urlStr: String? = null - try { - SslUtils.ignoreSsl() - urlStr = fileAttribute.url?.replace("+", "%20")?.replace(" ", "%20") - } catch (e: Exception) { - logger.error("Ignore SSL certificate exceptions", e) - } - val result = DownloadResult() + val urlStr = processUrl(fileAttribute) + val fileName = fileAttribute.fileName val realPath = getRelFilePath(fileName, fileAttribute.suffix!!, config.fileDir!!) - // 判断文件是否合法 - if (FileUtils.isIllegalFileName(realPath)) { - result.apply { - code = DownloadResult.CODE_FAIL - msg = "Download failed,The file name is invalid,$fileName" - } - return result - } - - if (!FileUtils.isAllowedUpload(realPath, config.prohibitSuffix)) { - result.apply { - code = DownloadResult.CODE_FAIL - msg = "Download Failed,Unsupported Type,$urlStr" - } - return result - } + if (!isValidFile(realPath, fileName!!, urlStr!!, config, result)) return result try { val url = WebUtils.normalizedURL(urlStr) if (FileUtils.isHttpUrl(url)) { - val realFile = File(realPath) - factory.setReadTimeout(72000) - factory.setConnectTimeout(10000) - factory.setConnectionRequestTimeout(2000) - factory.httpClient = HttpClientBuilder.create() - .setRedirectStrategy(DefaultRedirectStrategy()) - .build() - restTemplate.requestFactory = factory - val requestCallback = RequestCallback { request -> - request.headers.setAccept(listOf(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL)) - } - - try { - restTemplate.execute(url.toURI(), HttpMethod.GET, requestCallback) { fileResponse -> - org.apache.commons.io.FileUtils.copyToFile(fileResponse.body, realFile) - null - } - } catch (e: Exception) { - logger.error("The download failed: $e") - result.apply { - code = DownloadResult.CODE_FAIL - msg = "The download failed: $e" - } - return result - } + downloadHttpFile(url, realPath, result) } else if (FileUtils.isFtpUrl(url)) { - val ftpUsername = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_USERNAME) - val ftpPassword = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_PASSWORD) - val ftpControlEncoding = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_CONTROL_ENCODING) - FtpUtils.download( - fileAttribute.url, - realPath, - ftpUsername.takeUnless { it.isNullOrEmpty() } ?: config.ftpUsername, - ftpPassword.takeUnless { it.isNullOrEmpty() } ?: config.ftpPassword, - ftpControlEncoding.takeUnless { it.isNullOrEmpty() } ?: config.ftpControlEncoding - ) + downloadFtpFile(fileAttribute, realPath, config, result) } else { result.apply { code = DownloadResult.CODE_FAIL @@ -142,21 +89,7 @@ object DownloadUtils { } } - FileUtils.getFileMd5AndSize(realPath)?.let { (md5, size) -> - result.apply { - code = DownloadResult.CODE_SUCCESS - filePath = realPath - this.md5 = md5 - this.size = size - msg = fileName - } - } ?: run { - result.apply { - code = DownloadResult.CODE_FAIL - msg = "Failed to calculate MD5 or size for file: $fileName" - } - } - return result + return calculateFileMd5AndSize(realPath, fileName, result) } catch (e: IOException) { logger.error("File download failed，url：$urlStr") result.apply { @@ -170,6 +103,108 @@ object DownloadUtils { } } + private fun downloadHttpFile(url: URL, realPath: String, result: DownloadResult) { + try { + val realFile = File(realPath) + factory.setReadTimeout(72000) + factory.setConnectTimeout(10000) + factory.setConnectionRequestTimeout(2000) + factory.httpClient = HttpClientBuilder.create() + .setRedirectStrategy(DefaultRedirectStrategy()) + .build() + restTemplate.requestFactory = factory + val requestCallback = RequestCallback { request -> + request.headers.setAccept(listOf(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL)) + } + + restTemplate.execute(url.toURI(), HttpMethod.GET, requestCallback) { fileResponse -> + org.apache.commons.io.FileUtils.copyToFile(fileResponse.body, realFile) + null + } + } catch (e: Exception) { + logger.error("The download failed: $e") + result.apply { + code = DownloadResult.CODE_FAIL + msg = "The download failed: $e" + } + } + } + + private fun downloadFtpFile(fileAttribute: FileAttribute, + realPath: String, + config: PreviewConfig, + result: DownloadResult) { + try { + val ftpUsername = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_USERNAME) + val ftpPassword = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_PASSWORD) + val ftpControlEncoding = WebUtils.getUrlParameterReg(fileAttribute.url!!, URL_PARAM_FTP_CONTROL_ENCODING) + FtpUtils.download( + fileAttribute.url, + realPath, + ftpUsername.takeUnless { it.isNullOrEmpty() } ?: config.ftpUsername, + ftpPassword.takeUnless { it.isNullOrEmpty() } ?: config.ftpPassword, + ftpControlEncoding.takeUnless { it.isNullOrEmpty() } ?: config.ftpControlEncoding + ) + } catch (e: Exception) { + logger.error("FTP download failed: $e") + result.apply { + code = DownloadResult.CODE_FAIL + msg = "FTP download failed: $e" + } + } + } + + private fun calculateFileMd5AndSize(realPath: String, fileName: String, result: DownloadResult): DownloadResult { + FileUtils.getFileMd5AndSize(realPath)?.let { (md5, size) -> + result.apply { + code = DownloadResult.CODE_SUCCESS + filePath = realPath + this.md5 = md5 + this.size = size + msg = fileName + } + } ?: run { + result.apply { + code = DownloadResult.CODE_FAIL + msg = "Failed to calculate MD5 or size for file: $fileName" + } + } + return result + } + + private fun isValidFile(realPath: String, + fileName: String, + urlStr: String, + config: PreviewConfig, + result: DownloadResult): Boolean { + if (FileUtils.isIllegalFileName(realPath)) { + result.apply { + code = DownloadResult.CODE_FAIL + msg = "Download failed, The file name is invalid,$fileName" + } + return false + } + if (!FileUtils.isAllowedUpload(realPath, config.prohibitSuffix)) { + result.apply { + code = DownloadResult.CODE_FAIL + msg = "Download Failed, Unsupported Type, $urlStr" + } + return false + } + return true + } + + private fun processUrl(fileAttribute: FileAttribute): String? { + return try { + SslUtils.ignoreSsl() + fileAttribute.url?.replace("+", "%20")?.replace(" ", "%20") + } catch (e: Exception) { + logger.error("Ignore SSL certificate exceptions", e) + null + } + } + + /** * 文件存放的真实路径 * @param fileName 文件名 diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FileUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FileUtils.kt index 096a109f0d..646d208368 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FileUtils.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/FileUtils.kt @@ -38,8 +38,7 @@ import org.springframework.util.StringUtils import org.springframework.web.util.HtmlUtils import java.io.File import java.net.URL -import java.util.* -import kotlin.math.log +import java.util.Locale object FileUtils { private val logger = LoggerFactory.getLogger(FileUtils::class.java) diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt index e5b27c27b4..34b45abc07 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt @@ -32,7 +32,11 @@ package com.tencent.bkrepo.preview.utils import java.security.cert.X509Certificate -import javax.net.ssl.* +import javax.net.ssl.HostnameVerifier +import javax.net.ssl.HttpsURLConnection +import javax.net.ssl.TrustManager +import javax.net.ssl.X509TrustManager +import javax.net.ssl.SSLContext /** * SSL工具 @@ -65,8 +69,12 @@ object SslUtils { private class miTM : X509TrustManager { override fun getAcceptedIssuers(): Array? = null - override fun checkServerTrusted(certs: Array, authType: String) {} + override fun checkServerTrusted(certs: Array, authType: String) { + // No-op: Accepting all certificates + } - override fun checkClientTrusted(certs: Array, authType: String) {} + override fun checkClientTrusted(certs: Array, authType: String) { + // No-op: Accepting all certificates + } } } \ No newline at end of file