From 9659fdfcd51b7c33c2dfff7e23ae762a15754090 Mon Sep 17 00:00:00 2001 From: felixncheng Date: Tue, 19 Dec 2023 10:47:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=B3=BB=E7=BB=9FGC=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=20(#1557)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 支持系统GC和优化归档代码 #1483 * feat: 支持系统GC和优化归档代码 #1483 * feat: 修改metrics指标 #1483 --- .../tencent/bkrepo/archive/ArchiveStatus.kt | 10 +- .../tencent/bkrepo/archive/CompressStatus.kt | 53 +++++ .../bkrepo/archive/api/ArchiveClient.kt | 20 ++ .../archive/constant/ArchiveConstants.kt | 1 + .../bkrepo/archive/pojo/CompressedInfo.kt | 10 + .../request/CompleteCompressRequest.kt | 9 + .../archive/request/CompressFileRequest.kt | 11 + .../archive/request/DeleteCompressRequest.kt | 9 + .../archive/request/UncompressFileRequest.kt | 9 + .../archive/config/ArchiveProperties.kt | 3 + .../controller/service/ArchiveController.kt | 29 +++ .../archive/event/FileCompressedEvent.kt | 2 +- .../event/StorageFileCompressedEvent.kt | 15 ++ .../event/StorageFileUncompressedEvent.kt | 14 ++ .../bkrepo/archive/job/ArchiveSubscriber.kt | 98 -------- .../bkrepo/archive/job/BaseJobSubscriber.kt | 96 ++++++++ .../tencent/bkrepo/archive/job/JobContext.kt | 15 ++ .../bkrepo/archive/job/JobProcessMonitor.kt | 24 +- .../AbstractArchiveFileWrapperCallback.kt | 2 +- .../job/{ => archive}/ArchiveFileWrapper.kt | 2 +- .../ArchiveFileWrapperCallback.kt | 2 +- .../ArchiveJob.kt} | 73 +----- .../archive/job/archive/ArchiveSubscriber.kt | 57 +++++ .../job/{ => archive}/DiskHealthObserver.kt | 2 +- .../job/{ => archive}/FileCompressor.kt | 7 +- .../job/{ => archive}/FileDownloader.kt | 2 +- .../archive/job/{ => archive}/FileUploader.kt | 2 +- .../bkrepo/archive/job/archive/RestoreJob.kt | 53 +++++ .../RestoreSubscriber.kt} | 114 +++------- .../archive/job/compress/CompressJob.kt | 49 ++++ .../job/compress/CompressSubscriber.kt | 82 +++++++ .../archive/job/compress/UncompressJob.kt | 49 ++++ .../job/compress/UncompressSubscriber.kt | 67 ++++++ .../archive/listener/ArchiveListener.kt | 26 ++- .../listener/StorageCompressListener.kt | 65 ++++++ .../bkrepo/archive/metrics/ArchiveMetrics.kt | 135 +++++++---- .../bkrepo/archive/model/AbstractEntity.kt | 14 ++ .../tencent/bkrepo/archive/model/IdEntity.kt | 8 + .../bkrepo/archive/model/TArchiveFile.kt | 19 +- .../bkrepo/archive/model/TCompressFile.kt | 45 ++++ .../archive/repository/CompressFileDao.kt | 8 + .../repository/CompressFileRepository.kt | 12 + .../archive/service/ArchiveServiceImpl.kt | 12 +- .../bkrepo/archive/service/CompressService.kt | 33 +++ .../archive/service/CompressServiceImpl.kt | 150 +++++++++++++ .../bkrepo/archive/utils/ArchiveDaoUtils.kt | 37 +++ .../bkrepo/archive/utils/ArchiveUtils.kt | 38 ++++ .../bkrepo/archive/utils/ReactiveDaoUtils.kt | 66 ++++++ .../common/common-api/build.gradle.kts | 1 + .../api/collection/CollectionExtensions.kt | 43 ++++ .../bkrepo/common/api/constant/StringPool.kt | 7 + .../common/api/message/CommonMessageCode.kt | 8 +- .../common/api/stream/StreamExtensions.kt | 22 ++ .../bkrepo/common/api/util/StreamUtils.kt | 32 +++ .../bkrepo/common/artifact/stream/Range.kt | 20 +- .../common/artifact/stream/RangeTest.kt | 4 + .../manager/NodeResourceFactoryImpl.kt | 11 +- .../common/artifact/manager/StorageManager.kt | 16 -- .../manager/resource/LocalNodeResource.kt | 58 ++++- .../repository/redirect/CosRedirectService.kt | 13 +- .../common/common-bksync/build.gradle.kts | 1 + .../tencent/bkrepo/common/bksync/BkSync.kt | 30 +-- .../bkrepo/common/bksync/file/BDUtils.kt | 155 +++++++++++++ .../bksync/file/BkSyncDeltaFileHeader.kt | 22 ++ .../common/bksync/file/BkSyncDeltaSource.kt | 189 ++++++++++++++++ .../bksync/file/ByteArrayBkSyncDeltaSource.kt | 21 ++ .../bksync/file/FileBkSyncDeltaSource.kt | 22 ++ .../exception/TooLowerReuseRateException.kt | 6 + .../bkrepo/common/bksync/file/BDUtilsTest.kt | 54 +++++ .../bksync/file/BkSyncDeltaSourceTest.kt | 97 ++++++++ .../resources/i18n/messages_en.properties | 1 + .../resources/i18n/messages_zh_CN.properties | 1 + .../resources/i18n/messages_zh_TW.properties | 1 + .../storage/config/CompressProperties.kt | 8 + .../credentials/FileSystemCredentials.kt | 6 +- .../storage/credentials/HDFSCredentials.kt | 6 +- .../credentials/InnerCosCredentials.kt | 6 +- .../storage/credentials/S3Credentials.kt | 6 +- .../storage/credentials/StorageCredentials.kt | 4 +- .../storage/message/StorageMessageCode.kt | 5 +- .../core/AbstractEncryptorFileStorage.kt | 6 +- .../storage/core/AbstractFileStorage.kt | 6 +- .../storage/core/AbstractStorageService.kt | 4 +- .../common/storage/core/CompressSupport.kt | 212 ++++++++++++++++++ .../common/storage/core/StorageService.kt | 10 +- .../storage/core/cache/CacheStorageService.kt | 3 +- .../core/operation/CompressOperation.kt | 71 ++++++ .../storage/core/overlay/OverlayRangeUtils.kt | 12 +- .../storage/innercos/InnerCosFileStorage.kt | 6 +- .../bkrepo/common/storage/s3/S3Storage.kt | 6 +- .../core/cache/CacheStorageServiceTest.kt | 106 +++++++++ .../monitor/StorageHealthMonitorHelperTest.kt | 3 +- .../server/handler/FileOperationsHandler.kt | 2 +- .../job/batch/FileReferenceCleanupJob.kt | 15 +- .../bkrepo/job/batch/NodeCompressedJob.kt | 103 +++++++++ .../bkrepo/job/batch/NodeUncompressedJob.kt | 95 ++++++++ .../tencent/bkrepo/job/batch/SystemGcJob.kt | 170 ++++++++++++++ .../bkrepo/job/batch/utils/NodeCommonUtils.kt | 14 ++ .../properties/NodeCompressedJobProperties.kt | 35 +++ .../NodeUncompressedJobProperties.kt | 35 +++ .../properties/SystemGcJobProperties.kt | 68 ++++++ .../bkrepo/repository/api/NodeClient.kt | 66 ++++-- .../bkrepo/repository/pojo/node/NodeDetail.kt | 2 + .../bkrepo/repository/pojo/node/NodeInfo.kt | 2 + .../node/service/NodeCompressedRequest.kt | 16 ++ .../node/service/NodeUnCompressedRequest.kt | 16 ++ .../controller/service/NodeController.kt | 30 ++- .../tencent/bkrepo/repository/model/TNode.kt | 3 +- .../service/node/NodeCompressOperation.kt | 19 ++ .../repository/service/node/NodeService.kt | 3 +- .../service/node/impl/NodeBaseService.kt | 30 +-- .../service/node/impl/NodeCompressSupport.kt | 37 +++ .../service/node/impl/NodeServiceImpl.kt | 24 +- .../node/impl/edge/EdgeNodeServiceImpl.kt | 11 + 114 files changed, 3310 insertions(+), 466 deletions(-) create mode 100644 src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/CompressStatus.kt create mode 100644 src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt create mode 100644 src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompleteCompressRequest.kt create mode 100644 src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompressFileRequest.kt create mode 100644 src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/DeleteCompressRequest.kt create mode 100644 src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/UncompressFileRequest.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileCompressedEvent.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileUncompressedEvent.kt delete mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveSubscriber.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/BaseJobSubscriber.kt rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{ => archive}/AbstractArchiveFileWrapperCallback.kt (94%) rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{ => archive}/ArchiveFileWrapper.kt (92%) rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{ => archive}/ArchiveFileWrapperCallback.kt (81%) rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{ReactiveArchiveJob.kt => archive/ArchiveJob.kt} (68%) create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveSubscriber.kt rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{ => archive}/DiskHealthObserver.kt (69%) rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{ => archive}/FileCompressor.kt (96%) rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{ => archive}/FileDownloader.kt (99%) rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{ => archive}/FileUploader.kt (97%) create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/RestoreJob.kt rename src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/{RestoreJob.kt => archive/RestoreSubscriber.kt} (57%) create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/CompressJob.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/CompressSubscriber.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/UncompressJob.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/UncompressSubscriber.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/listener/StorageCompressListener.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/AbstractEntity.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/IdEntity.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/TCompressFile.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/repository/CompressFileDao.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/repository/CompressFileRepository.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveDaoUtils.kt create mode 100644 src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ReactiveDaoUtils.kt create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/stream/StreamExtensions.kt create mode 100644 src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BDUtils.kt create mode 100644 src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaFileHeader.kt create mode 100644 src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaSource.kt create mode 100644 src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/ByteArrayBkSyncDeltaSource.kt create mode 100644 src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/FileBkSyncDeltaSource.kt create mode 100644 src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/transfer/exception/TooLowerReuseRateException.kt create mode 100644 src/backend/common/common-bksync/src/test/kotlin/com/tencent/bkrepo/common/bksync/file/BDUtilsTest.kt create mode 100644 src/backend/common/common-bksync/src/test/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaSourceTest.kt create mode 100644 src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/CompressProperties.kt create mode 100644 src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/CompressSupport.kt create mode 100644 src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/operation/CompressOperation.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/NodeCompressedJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/NodeUncompressedJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/SystemGcJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/NodeCompressedJobProperties.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/NodeUncompressedJobProperties.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/SystemGcJobProperties.kt create mode 100644 src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeCompressedRequest.kt create mode 100644 src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeUnCompressedRequest.kt create mode 100644 src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeCompressOperation.kt create mode 100644 src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeCompressSupport.kt diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/ArchiveStatus.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/ArchiveStatus.kt index eb1de1dc9e..5ba217cb75 100644 --- a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/ArchiveStatus.kt +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/ArchiveStatus.kt @@ -16,6 +16,11 @@ enum class ArchiveStatus { * */ ARCHIVED, + /** + * 已完成 + * */ + COMPLETED, + /** * 待恢复 * */ @@ -40,9 +45,4 @@ enum class ArchiveStatus { * 恢复失败 * */ RESTORE_FAILED, - - /** - * 已完成 - * */ - COMPLETED, } diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/CompressStatus.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/CompressStatus.kt new file mode 100644 index 0000000000..36b753516a --- /dev/null +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/CompressStatus.kt @@ -0,0 +1,53 @@ +package com.tencent.bkrepo.archive + +enum class CompressStatus { + /** + * 已创建 + * */ + CREATED, + + /** + * 压缩中 + * */ + COMPRESSING, + + /** + * 已压缩 + * */ + COMPRESSED, + + /** + * 已完成 + * */ + COMPLETED, + + /** + * 等待解压 + * */ + WAIT_TO_UNCOMPRESS, + + /** + * 正在解压 + * */ + UNCOMPRESSING, + + /** + * 已解压 + * */ + UNCOMPRESSED, + + /** + * 表示链头 + * */ + NONE, + + /** + * 压缩失败 + * */ + COMPRESS_FAILED, + + /** + * 解压失败 + * */ + UNCOMPRESS_FAILED, +} diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt index 0eb6952088..fcb790384f 100644 --- a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt @@ -2,17 +2,25 @@ package com.tencent.bkrepo.archive.api import com.tencent.bkrepo.archive.pojo.ArchiveFile import com.tencent.bkrepo.archive.request.ArchiveFileRequest +import com.tencent.bkrepo.archive.request.CompleteCompressRequest +import com.tencent.bkrepo.archive.request.CompressFileRequest import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest +import com.tencent.bkrepo.archive.request.DeleteCompressRequest +import com.tencent.bkrepo.archive.request.UncompressFileRequest import com.tencent.bkrepo.common.api.constant.ARCHIVE_SERVICE_NAME import com.tencent.bkrepo.common.api.pojo.Response import org.springframework.cloud.openfeign.FeignClient import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam +/** + * 归档服务接口 + * */ @FeignClient(ARCHIVE_SERVICE_NAME) @RequestMapping("/service/archive") interface ArchiveClient { @@ -33,4 +41,16 @@ interface ArchiveClient { @RequestParam sha256: String, @RequestParam(required = false) storageCredentialsKey: String?, ): Response + + @PutMapping("/compress") + fun compress(@RequestBody request: CompressFileRequest): Response + + @PutMapping("/uncompress") + fun uncompress(@RequestBody request: UncompressFileRequest): Response + + @DeleteMapping("/compress") + fun deleteCompress(@RequestBody request: DeleteCompressRequest): Response + + @PutMapping("/compress/complete") + fun completeCompress(@RequestBody request: CompleteCompressRequest): Response } diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/constant/ArchiveConstants.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/constant/ArchiveConstants.kt index 4e537eccdb..3cadf7e360 100644 --- a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/constant/ArchiveConstants.kt +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/constant/ArchiveConstants.kt @@ -2,3 +2,4 @@ package com.tencent.bkrepo.archive.constant const val XZ_SUFFIX = ".xz" const val DEEP_ARCHIVE = "DEEP_ARCHIVE" +const val DEFAULT_KEY = "default" diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt new file mode 100644 index 0000000000..4461f8c022 --- /dev/null +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt @@ -0,0 +1,10 @@ +package com.tencent.bkrepo.archive.pojo + +import com.tencent.bkrepo.common.api.constant.StringPool + +class CompressedInfo( + val status: Int, // 0 压缩中， 1 压缩成功 + val uncompressedSize: Long, + val compressedSize: Long, + val ratio: String = StringPool.calculateRatio(uncompressedSize, compressedSize), +) diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompleteCompressRequest.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompleteCompressRequest.kt new file mode 100644 index 0000000000..d49ce35da9 --- /dev/null +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompleteCompressRequest.kt @@ -0,0 +1,9 @@ +package com.tencent.bkrepo.archive.request + +import com.tencent.bkrepo.repository.constant.SYSTEM_USER + +data class CompleteCompressRequest( + val sha256: String, + val storageCredentialsKey: String?, + val operator: String = SYSTEM_USER, +) diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompressFileRequest.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompressFileRequest.kt new file mode 100644 index 0000000000..0522b6b11d --- /dev/null +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompressFileRequest.kt @@ -0,0 +1,11 @@ +package com.tencent.bkrepo.archive.request + +import com.tencent.bkrepo.repository.constant.SYSTEM_USER + +data class CompressFileRequest( + val sha256: String, + val size: Long, + val baseSha256: String, + val storageCredentialsKey: String?, + val operator: String = SYSTEM_USER, +) diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/DeleteCompressRequest.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/DeleteCompressRequest.kt new file mode 100644 index 0000000000..aa8520a120 --- /dev/null +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/DeleteCompressRequest.kt @@ -0,0 +1,9 @@ +package com.tencent.bkrepo.archive.request + +import com.tencent.bkrepo.repository.constant.SYSTEM_USER + +data class DeleteCompressRequest( + val sha256: String, + val storageCredentialsKey: String?, + val operator: String = SYSTEM_USER, +) diff --git a/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/UncompressFileRequest.kt b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/UncompressFileRequest.kt new file mode 100644 index 0000000000..8ce03a5ca9 --- /dev/null +++ b/src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/UncompressFileRequest.kt @@ -0,0 +1,9 @@ +package com.tencent.bkrepo.archive.request + +import com.tencent.bkrepo.repository.constant.SYSTEM_USER + +data class UncompressFileRequest( + val sha256: String, + val storageCredentialsKey: String?, + val operator: String = SYSTEM_USER, +) diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/config/ArchiveProperties.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/config/ArchiveProperties.kt index 80fb22682f..d77d8282db 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/config/ArchiveProperties.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/config/ArchiveProperties.kt @@ -5,6 +5,9 @@ import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.boot.context.properties.NestedConfigurationProperty import org.springframework.util.unit.DataSize +/** + * 归档服务配置 + * */ @ConfigurationProperties("archive") data class ArchiveProperties( @NestedConfigurationProperty diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt index ec1ce5062e..9018cf1310 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt @@ -3,15 +3,24 @@ package com.tencent.bkrepo.archive.controller.service import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.pojo.ArchiveFile import com.tencent.bkrepo.archive.request.ArchiveFileRequest +import com.tencent.bkrepo.archive.request.CompleteCompressRequest +import com.tencent.bkrepo.archive.request.CompressFileRequest import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest +import com.tencent.bkrepo.archive.request.DeleteCompressRequest +import com.tencent.bkrepo.archive.request.UncompressFileRequest import com.tencent.bkrepo.archive.service.ArchiveService +import com.tencent.bkrepo.archive.service.CompressService import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.service.util.ResponseBuilder import org.springframework.web.bind.annotation.RestController +/** + * 归档服务控制器 + * */ @RestController class ArchiveController( private val archiveService: ArchiveService, + private val compressService: CompressService, ) : ArchiveClient { override fun archive(request: CreateArchiveFileRequest): Response { archiveService.archive(request) @@ -36,4 +45,24 @@ class ArchiveController( archiveService.complete(request) return ResponseBuilder.success() } + + override fun compress(request: CompressFileRequest): Response { + compressService.compress(request) + return ResponseBuilder.success() + } + + override fun uncompress(request: UncompressFileRequest): Response { + compressService.uncompress(request) + return ResponseBuilder.success() + } + + override fun deleteCompress(request: DeleteCompressRequest): Response { + compressService.delete(request) + return ResponseBuilder.success() + } + + override fun completeCompress(request: CompleteCompressRequest): Response { + compressService.complete(request) + return ResponseBuilder.success() + } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/FileCompressedEvent.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/FileCompressedEvent.kt index e8a31e63ce..28de55a014 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/FileCompressedEvent.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/FileCompressedEvent.kt @@ -3,7 +3,7 @@ package com.tencent.bkrepo.archive.event import com.tencent.bkrepo.common.storage.monitor.Throughput /** - * 文件归档事件 + * 归档压缩事件 * */ data class FileCompressedEvent( val sha256: String, diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileCompressedEvent.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileCompressedEvent.kt new file mode 100644 index 0000000000..1647ba7825 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileCompressedEvent.kt @@ -0,0 +1,15 @@ +package com.tencent.bkrepo.archive.event + +import com.tencent.bkrepo.common.storage.monitor.Throughput + +/** + * 存储文件压缩事件 + * */ +data class StorageFileCompressedEvent( + val sha256: String, + val baseSha256: String, + val uncompressed: Long, + val compressed: Long, + val storageCredentialsKey: String?, + val throughput: Throughput, +) diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileUncompressedEvent.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileUncompressedEvent.kt new file mode 100644 index 0000000000..fd77c22b03 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileUncompressedEvent.kt @@ -0,0 +1,14 @@ +package com.tencent.bkrepo.archive.event + +import com.tencent.bkrepo.common.storage.monitor.Throughput + +/** + * 存储文件解压事件 + * */ +data class StorageFileUncompressedEvent( + val sha256: String, + val uncompressed: Long, + val compressed: Long, + val storageCredentialsKey: String?, + val throughput: Throughput, +) diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveSubscriber.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveSubscriber.kt deleted file mode 100644 index e8654afdb4..0000000000 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveSubscriber.kt +++ /dev/null @@ -1,98 +0,0 @@ -package com.tencent.bkrepo.archive.job - -import com.tencent.bkrepo.archive.ArchiveStatus -import com.tencent.bkrepo.archive.event.FileArchivedEvent -import com.tencent.bkrepo.archive.extensions.key -import com.tencent.bkrepo.archive.model.TArchiveFile -import com.tencent.bkrepo.archive.repository.ArchiveFileRepository -import com.tencent.bkrepo.archive.utils.ArchiveUtils -import com.tencent.bkrepo.common.service.util.SpringContextUtils -import com.tencent.bkrepo.common.storage.monitor.Throughput -import java.time.Duration -import java.time.LocalDateTime -import java.time.temporal.ChronoUnit -import java.util.concurrent.CountDownLatch -import org.reactivestreams.Subscription -import org.slf4j.LoggerFactory -import reactor.core.publisher.BaseSubscriber -import reactor.core.publisher.SignalType - -/** - * 归档任务订阅者 - * */ -class ArchiveSubscriber(val archiveFileRepository: ArchiveFileRepository) : BaseSubscriber() { - /** - * 开始时间 - * */ - private var startAt: Long = -1L - - /** - * 任务上下文 - * */ - private val jobContext = JobContext() - - /** - * 闭锁，用于同步任务 - * */ - private val countDownLatch = CountDownLatch(1) - override fun hookOnSubscribe(subscription: Subscription) { - logger.info("Start execute archive job.") - ArchiveUtils.monitor.addMonitor(MONITOR_ID, jobContext) - startAt = System.currentTimeMillis() - super.hookOnSubscribe(subscription) - } - - override fun hookOnNext(fileWrapper: ArchiveFileWrapper) { - jobContext.total.incrementAndGet() - val archiveFile = fileWrapper.archiveFile - jobContext.totalSize.addAndGet(archiveFile.size) - if (fileWrapper.throwable != null) { - // error - logger.error("Archive file failed: ", fileWrapper.throwable) - jobContext.failed.incrementAndGet() - updateArchiveFile(archiveFile, ArchiveStatus.ARCHIVE_FAILED) - } else { - // success - logger.info("Archive file(${archiveFile.key()}) successful") - jobContext.success.incrementAndGet() - val nanos = Duration.between(fileWrapper.startTime, LocalDateTime.now()).toNanos() - val tp = Throughput(archiveFile.size, nanos) - val event = FileArchivedEvent(archiveFile.sha256, archiveFile.storageCredentialsKey, tp) - SpringContextUtils.publishEvent(event) - updateArchiveFile(archiveFile, ArchiveStatus.ARCHIVED) - } - } - - override fun hookOnComplete() { - val stopAt = System.currentTimeMillis() - val throughput = Throughput(jobContext.totalSize.get(), stopAt - startAt, ChronoUnit.MILLIS) - logger.info("Archive job execute successful.summary: $jobContext $throughput.") - } - - override fun hookOnError(throwable: Throwable) { - logger.error("Archive job execute successful failed: ", throwable) - } - - override fun hookFinally(type: SignalType) { - ArchiveUtils.monitor.removeMonitor(MONITOR_ID) - countDownLatch.countDown() - } - - fun block() { - countDownLatch.await() - } - - /** - * 更新文件状态 - * */ - private fun updateArchiveFile(file: TArchiveFile, status: ArchiveStatus) { - file.lastModifiedDate = LocalDateTime.now() - file.status = status - archiveFileRepository.save(file) - } - - companion object { - private val logger = LoggerFactory.getLogger(ArchiveSubscriber::class.java) - private const val MONITOR_ID = "archive-job" - } -} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/BaseJobSubscriber.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/BaseJobSubscriber.kt new file mode 100644 index 0000000000..a421b153c0 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/BaseJobSubscriber.kt @@ -0,0 +1,96 @@ +package com.tencent.bkrepo.archive.job + +import com.tencent.bkrepo.archive.utils.ArchiveUtils +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.common.storage.monitor.Throughput +import org.reactivestreams.Subscription +import reactor.core.publisher.BaseSubscriber +import reactor.core.publisher.SignalType +import reactor.util.context.Context +import java.time.temporal.ChronoUnit +import java.util.concurrent.CountDownLatch + +/** + * 通用任务订阅者 + * 完成任务的通用功能，比如context的维护，任务的起始记录等 + * */ +open class BaseJobSubscriber : BaseSubscriber() { + + /** + * 开始时间 + * */ + private var startAt: Long = -1L + + /** + * 任务上下文 + * */ + protected val jobContext = JobContext() + + private fun getJobName(): String = javaClass.simpleName + + /** + * 闭锁，用于同步任务 + * */ + private val countDownLatch = CountDownLatch(1) + + override fun hookOnSubscribe(subscription: Subscription) { + LoggerHolder.jobLogger.info("Start execute job[${getJobName()}].") + startAt = System.currentTimeMillis() + ArchiveUtils.monitor.addMonitor(getJobName(), jobContext) + super.hookOnSubscribe(subscription) + } + + override fun hookOnNext(value: T) { + try { + doOnNext(value) + jobContext.success.incrementAndGet() + } catch (e: Exception) { + LoggerHolder.jobLogger.error("DoOnNext error: ", e) + jobContext.failed.incrementAndGet() + } finally { + jobContext.total.incrementAndGet() + jobContext.totalSize.addAndGet(getSize(value)) + } + } + + /** + * 处理[value] + * */ + protected open fun doOnNext(value: T) { + } + + /** + * 获取[value]的大小 + * */ + protected open fun getSize(value: T): Long { + return 0 + } + + override fun hookOnComplete() { + val stopAt = System.currentTimeMillis() + val throughput = Throughput(jobContext.totalSize.get(), stopAt - startAt, ChronoUnit.MILLIS) + LoggerHolder.jobLogger.info("Job[${getJobName()}] execute successful.summary: $jobContext $throughput.") + } + + override fun hookOnError(throwable: Throwable) { + LoggerHolder.jobLogger.error("Job[${getJobName()}] execute failed: ", throwable) + } + + override fun hookFinally(type: SignalType) { + ArchiveUtils.monitor.removeMonitor(getJobName()) + countDownLatch.countDown() + } + + override fun currentContext(): Context { + return Context.of(JOB_NAME, getJobName(), JOB_CTX, jobContext) + } + + fun blockLast() { + countDownLatch.await() + } + + companion object { + const val JOB_NAME = "JOB-NAME" + const val JOB_CTX = "JOB-CTX" + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/JobContext.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/JobContext.kt index 55b37eb7f7..fbf532e1d8 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/JobContext.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/JobContext.kt @@ -3,10 +3,25 @@ package com.tencent.bkrepo.archive.job import com.tencent.bkrepo.common.api.util.HumanReadable import java.util.concurrent.atomic.AtomicLong +/** + * 任务上下文 + * */ data class JobContext( + /** + * 任务成功数 + * */ var success: AtomicLong = AtomicLong(), + /** + * 任务失败数 + * */ var failed: AtomicLong = AtomicLong(), + /** + * 任务总数 + * */ var total: AtomicLong = AtomicLong(), + /** + * 任务处理总大小 + * */ var totalSize: AtomicLong = AtomicLong(), ) { fun reset() { diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/JobProcessMonitor.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/JobProcessMonitor.kt index 1bf8520267..268deeb592 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/JobProcessMonitor.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/JobProcessMonitor.kt @@ -4,20 +4,25 @@ import org.slf4j.LoggerFactory import kotlin.concurrent.thread /** - * 流程监控器 + * 任务进度监控器 + * 定时打印任务进度 * */ class JobProcessMonitor { - val contexts = mutableMapOf() + private val contexts = mutableMapOf() private var running = false + + /** + * 启动任务进度监控 + * */ fun start() { if (running) { return } running = true - thread(name = "job-process-monitor", isDaemon = true) { + thread(name = NAME, isDaemon = true) { while (running) { - Thread.sleep(2000) + Thread.sleep(TIME_INTERVAL) contexts.forEach { (id, context) -> logger.info("Process info($id): $context") } @@ -25,19 +30,30 @@ class JobProcessMonitor { } } + /** + * 停止任务进度监控 + * */ fun stop() { running = false } + /** + * 添加监控 + * */ fun addMonitor(id: String, context: JobContext) { contexts[id] = context } + /** + * 移除监控 + * */ fun removeMonitor(id: String) { contexts.remove(id) } companion object { private val logger = LoggerFactory.getLogger(JobProcessMonitor::class.java) + private const val TIME_INTERVAL = 2000L + private const val NAME = "job-process-monitor" } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/AbstractArchiveFileWrapperCallback.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/AbstractArchiveFileWrapperCallback.kt similarity index 94% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/AbstractArchiveFileWrapperCallback.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/AbstractArchiveFileWrapperCallback.kt index 163cb08076..2a4334beb7 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/AbstractArchiveFileWrapperCallback.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/AbstractArchiveFileWrapperCallback.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive import org.reactivestreams.Publisher import reactor.core.publisher.Mono diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveFileWrapper.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveFileWrapper.kt similarity index 92% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveFileWrapper.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveFileWrapper.kt index 675511797e..f105743aae 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveFileWrapper.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveFileWrapper.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive import com.tencent.bkrepo.archive.model.TArchiveFile import java.nio.file.Path diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveFileWrapperCallback.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveFileWrapperCallback.kt similarity index 81% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveFileWrapperCallback.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveFileWrapperCallback.kt index bf15f0828e..979085c1a7 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ArchiveFileWrapperCallback.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveFileWrapperCallback.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive import org.reactivestreams.Publisher diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ReactiveArchiveJob.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveJob.kt similarity index 68% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ReactiveArchiveJob.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveJob.kt index dfc7a01397..da8cf6c198 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/ReactiveArchiveJob.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveJob.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive import com.google.common.util.concurrent.ThreadFactoryBuilder import com.tencent.bkrepo.archive.ArchiveStatus @@ -6,21 +6,17 @@ import com.tencent.bkrepo.archive.config.ArchiveProperties import com.tencent.bkrepo.archive.extensions.key import com.tencent.bkrepo.archive.model.TArchiveFile import com.tencent.bkrepo.archive.repository.ArchiveFileRepository +import com.tencent.bkrepo.archive.utils.ArchiveUtils.Companion.newFixedAndCachedThreadPool +import com.tencent.bkrepo.archive.utils.ReactiveDaoUtils import com.tencent.bkrepo.common.mongo.constant.ID -import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.innercos.client.CosClient import java.nio.file.Paths import java.time.Duration import java.time.LocalDateTime -import java.util.concurrent.ArrayBlockingQueue -import java.util.concurrent.ThreadFactory -import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit import org.bson.types.ObjectId -import org.reactivestreams.Publisher import org.slf4j.LoggerFactory -import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.ReactiveMongoTemplate import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -30,7 +26,6 @@ import org.springframework.data.mongodb.core.query.where import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component import reactor.core.publisher.Flux -import reactor.core.publisher.FluxSink import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers @@ -38,7 +33,7 @@ import reactor.core.scheduler.Schedulers * 数据归档任务 * */ @Component -class ReactiveArchiveJob( +class ArchiveJob( private val reactiveMongoTemplate: ReactiveMongoTemplate, private val archiveProperties: ArchiveProperties, private val storageService: StorageService, @@ -73,7 +68,7 @@ class ReactiveArchiveJob( /** * 下载任务线程池 * */ - val httpDownloadPool = buildThreadPool( + val httpDownloadPool = newFixedAndCachedThreadPool( archiveProperties.ioThreads, ThreadFactoryBuilder().setNameFormat("archive-download-%d").build(), ) @@ -81,7 +76,7 @@ class ReactiveArchiveJob( /** * 压缩任务线程池 * */ - val compressPool = buildThreadPool( + val compressPool = newFixedAndCachedThreadPool( archiveProperties.compressThreads, ThreadFactoryBuilder().setNameFormat("archive-compress-%d").build(), ) @@ -89,7 +84,7 @@ class ReactiveArchiveJob( /** * 上传任务线程池 * */ - val httpUploadPool = buildThreadPool( + val httpUploadPool = newFixedAndCachedThreadPool( archiveProperties.ioThreads, ThreadFactoryBuilder().setNameFormat("archive-upload-%d").build(), ) @@ -119,20 +114,7 @@ class ReactiveArchiveJob( val criteria = where(TArchiveFile::status).isEqualTo(ArchiveStatus.CREATED) val query = Query.query(criteria) .limit(archiveProperties.queryLimit) - val idQueryCursor = IdQueryCursor(query, reactiveMongoTemplate) - return Flux.create { recurseCursor(idQueryCursor, it) } - } - - private fun recurseCursor(idQueryCursor: IdQueryCursor, sink: FluxSink) { - Mono.from(idQueryCursor.next()) - .doOnSuccess { results -> - results.forEach { sink.next(it) } - if (idQueryCursor.hasNext) { - recurseCursor(idQueryCursor, sink) - } else { - sink.complete() - } - }.subscribe() + return ReactiveDaoUtils.query(query, TArchiveFile::class.java) } /** @@ -173,46 +155,11 @@ class ReactiveArchiveJob( .runOn(Schedulers.fromExecutor(httpUploadPool), prefetch) .flatMap(uploader::onArchiveFileWrapper) // 上传 .subscribe(subscriber) - subscriber.block() - } - - /** - * id查询游标，将查询使用id在进行分页查找，避免大表skip，导致性能下降 - * */ - private class IdQueryCursor( - val query: Query, - val reactiveMongoTemplate: ReactiveMongoTemplate, - ) { - private var lastId = MIN_OBJECT_ID - var hasNext: Boolean = true - - fun next(): Publisher> { - val idQuery = Query.of(query).addCriteria(Criteria.where(ID).gt(ObjectId(lastId))) - .with(Sort.by(ID).ascending()) - return reactiveMongoTemplate.find(idQuery, TArchiveFile::class.java).collectList() - .doOnSuccess { - if (it.isNotEmpty()) { - lastId = it.last().id!! - } - hasNext = it.size == query.limit - } - } - } - - private fun buildThreadPool(threads: Int, threadFactory: ThreadFactory): ThreadPoolExecutor { - return ThreadPoolExecutor( - threads, - threads, - 60, - TimeUnit.SECONDS, - ArrayBlockingQueue(8096), - threadFactory, - ThreadPoolExecutor.CallerRunsPolicy(), - ) + subscriber.blockLast() } companion object { - private val logger = LoggerFactory.getLogger(ReactiveArchiveJob::class.java) + private val logger = LoggerFactory.getLogger(ArchiveJob::class.java) private const val DISK_CHECK_PERIOD = 3000L } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveSubscriber.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveSubscriber.kt new file mode 100644 index 0000000000..49654b950d --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/ArchiveSubscriber.kt @@ -0,0 +1,57 @@ +package com.tencent.bkrepo.archive.job.archive + +import com.tencent.bkrepo.archive.ArchiveStatus +import com.tencent.bkrepo.archive.event.FileArchivedEvent +import com.tencent.bkrepo.archive.extensions.key +import com.tencent.bkrepo.archive.job.BaseJobSubscriber +import com.tencent.bkrepo.archive.model.TArchiveFile +import com.tencent.bkrepo.archive.repository.ArchiveFileRepository +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.common.storage.monitor.Throughput +import java.time.Duration +import java.time.LocalDateTime +import org.slf4j.LoggerFactory + +/** + * 归档任务订阅者 + * */ +class ArchiveSubscriber( + private val archiveFileRepository: ArchiveFileRepository, +) : BaseJobSubscriber() { + + override fun doOnNext(value: ArchiveFileWrapper) { + with(value) { + jobContext.totalSize.addAndGet(archiveFile.size) + throwable?.let { + // error + logger.error("Archive file failed: ", throwable) + updateArchiveFile(archiveFile, ArchiveStatus.ARCHIVE_FAILED) + throw it + } + // success + logger.info("Archive file(${archiveFile.key()}) successful") + val nanos = Duration.between(startTime, LocalDateTime.now()).toNanos() + val tp = Throughput(archiveFile.size, nanos) + val event = FileArchivedEvent(archiveFile.sha256, archiveFile.storageCredentialsKey, tp) + SpringContextUtils.publishEvent(event) + updateArchiveFile(archiveFile, ArchiveStatus.ARCHIVED) + } + } + + /** + * 更新文件状态 + * */ + private fun updateArchiveFile(file: TArchiveFile, status: ArchiveStatus) { + file.lastModifiedDate = LocalDateTime.now() + file.status = status + archiveFileRepository.save(file) + } + + override fun getSize(value: ArchiveFileWrapper): Long { + return value.archiveFile.size + } + + companion object { + private val logger = LoggerFactory.getLogger(ArchiveSubscriber::class.java) + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/DiskHealthObserver.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/DiskHealthObserver.kt similarity index 69% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/DiskHealthObserver.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/DiskHealthObserver.kt index 676143179d..bcf4c03dd0 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/DiskHealthObserver.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/DiskHealthObserver.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive /** * 磁盘健康观察者 diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileCompressor.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileCompressor.kt similarity index 96% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileCompressor.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileCompressor.kt index f3694380a1..28d7e57125 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileCompressor.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileCompressor.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive import com.tencent.bkrepo.archive.constant.XZ_SUFFIX import com.tencent.bkrepo.archive.event.FileCompressedEvent @@ -28,10 +28,6 @@ class FileCompressor( * */ private val workPath: String, ) : AbstractArchiveFileWrapperCallback() { - /** - * 是否使用command - * */ - private val useCmd = ArchiveUtils.supportXZCmd() /** * 压缩文件路径 @@ -47,7 +43,6 @@ class FileCompressor( if (!Files.exists(compressedPath)) { Files.createDirectories(compressedPath) } - require(useCmd) } override fun process(fileWrapper: ArchiveFileWrapper): Mono { diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileDownloader.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileDownloader.kt similarity index 99% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileDownloader.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileDownloader.kt index aa2dcc1400..fa1ce70adc 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileDownloader.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileDownloader.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive import com.google.common.hash.HashCode import com.tencent.bkrepo.archive.constant.XZ_SUFFIX diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileUploader.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileUploader.kt similarity index 97% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileUploader.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileUploader.kt index 128819dcdb..a2bdfd32c5 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/FileUploader.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/FileUploader.kt @@ -1,4 +1,4 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive import com.tencent.bkrepo.archive.constant.DEEP_ARCHIVE import com.tencent.bkrepo.archive.constant.XZ_SUFFIX diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/RestoreJob.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/RestoreJob.kt new file mode 100644 index 0000000000..13735a506a --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/RestoreJob.kt @@ -0,0 +1,53 @@ +package com.tencent.bkrepo.archive.job.archive + +import com.tencent.bkrepo.archive.ArchiveStatus +import com.tencent.bkrepo.archive.config.ArchiveProperties +import com.tencent.bkrepo.archive.model.TArchiveFile +import com.tencent.bkrepo.archive.repository.ArchiveFileDao +import com.tencent.bkrepo.archive.repository.ArchiveFileRepository +import com.tencent.bkrepo.archive.utils.ReactiveDaoUtils +import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.common.storage.innercos.client.CosClient +import java.util.concurrent.TimeUnit +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component +import reactor.core.publisher.Flux + +/** + * 数据恢复任务 + * */ +@Component +class RestoreJob( + private val archiveFileRepository: ArchiveFileRepository, + private val storageService: StorageService, + private val archiveProperties: ArchiveProperties, + private val archiveFileDao: ArchiveFileDao, +) { + private val cosClient = CosClient(archiveProperties.cos) + + /** + * 获取待归档文件列表 + * */ + fun listFiles(): Flux { + val criteria = where(TArchiveFile::status).isEqualTo(ArchiveStatus.WAIT_TO_RESTORE) + val query = Query.query(criteria) + .limit(archiveProperties.queryLimit) + return ReactiveDaoUtils.query(query, TArchiveFile::class.java) + } + + @Scheduled(fixedDelay = 12, timeUnit = TimeUnit.HOURS) + fun restore() { + val subscriber = RestoreSubscriber( + cosClient, + archiveFileDao, + storageService, + archiveFileRepository, + archiveProperties.workDir, + ) + listFiles().subscribe(subscriber) + subscriber.blockLast() + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/RestoreJob.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/RestoreSubscriber.kt similarity index 57% rename from src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/RestoreJob.kt rename to src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/RestoreSubscriber.kt index e31e0d0216..187270f44a 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/RestoreJob.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/archive/RestoreSubscriber.kt @@ -1,58 +1,44 @@ -package com.tencent.bkrepo.archive.job +package com.tencent.bkrepo.archive.job.archive import com.tencent.bkrepo.archive.ArchiveStatus -import com.tencent.bkrepo.archive.config.ArchiveProperties import com.tencent.bkrepo.archive.constant.XZ_SUFFIX import com.tencent.bkrepo.archive.event.FileRestoredEvent import com.tencent.bkrepo.archive.extensions.key +import com.tencent.bkrepo.archive.job.BaseJobSubscriber import com.tencent.bkrepo.archive.model.TArchiveFile import com.tencent.bkrepo.archive.repository.ArchiveFileDao import com.tencent.bkrepo.archive.repository.ArchiveFileRepository -import com.tencent.bkrepo.archive.utils.ArchiveFileQueryHelper +import com.tencent.bkrepo.archive.utils.ArchiveDaoUtils.optimisticLock import com.tencent.bkrepo.archive.utils.ArchiveUtils import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.api.toArtifactFile -import com.tencent.bkrepo.common.mongo.constant.ID -import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.innercos.client.CosClient import com.tencent.bkrepo.common.storage.innercos.request.CheckObjectExistRequest import com.tencent.bkrepo.common.storage.innercos.request.GetObjectRequest import com.tencent.bkrepo.common.storage.monitor.Throughput +import org.slf4j.LoggerFactory import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.time.Duration import java.time.LocalDateTime -import java.util.concurrent.TimeUnit -import org.bson.types.ObjectId -import org.slf4j.LoggerFactory -import org.springframework.data.mongodb.core.query.Criteria -import org.springframework.data.mongodb.core.query.Query -import org.springframework.data.mongodb.core.query.Update -import org.springframework.data.mongodb.core.query.isEqualTo -import org.springframework.scheduling.annotation.Scheduled -import org.springframework.stereotype.Component /** - * 数据恢复任务 + * 数据恢复订阅者 + * 处理具体数据恢复 * */ -@Component -class RestoreJob( - private val archiveFileRepository: ArchiveFileRepository, - private val storageService: StorageService, - private val archiveProperties: ArchiveProperties, +class RestoreSubscriber( + private val cosClient: CosClient, private val archiveFileDao: ArchiveFileDao, -) { - private val cosClient = CosClient(archiveProperties.cos) - private val useCmd = ArchiveUtils.supportXZCmd() - private val workDir = archiveProperties.workDir + private val storageService: StorageService, + private val archiveFileRepository: ArchiveFileRepository, + workDir: String, +) : BaseJobSubscriber() { private val tempPath: Path = Paths.get(workDir, "temp") private val compressedPath: Path = Paths.get(workDir, "compressed") private val filesPath: Path = Paths.get(workDir, "files") - val context: JobContext = JobContext() - private val monitorId = "restore-job" init { if (!Files.exists(tempPath)) { @@ -66,58 +52,18 @@ class RestoreJob( } } - @Scheduled(fixedDelay = 12, timeUnit = TimeUnit.HOURS) - fun restore() { - logger.info("Begin restore.") - val startAt = System.nanoTime() - context.reset() - ArchiveUtils.monitor.addMonitor(monitorId, context) - var lastId = MIN_OBJECT_ID - var query = ArchiveFileQueryHelper - .buildQuery(ArchiveStatus.WAIT_TO_RESTORE, lastId, archiveProperties.queryLimit) - var files = archiveFileDao.find(query).toMutableList() - while (files.isNotEmpty()) { - logger.info("Find ${files.size} file to restore.") - lastId = files.last().id!! - files.shuffled() - restoreFiles(files) - query = - ArchiveFileQueryHelper.buildQuery(ArchiveStatus.WAIT_TO_RESTORE, lastId, archiveProperties.queryLimit) - files = archiveFileDao.find(query).toMutableList() - } - ArchiveUtils.monitor.removeMonitor(monitorId) - val stopAt = System.nanoTime() - val throughput = Throughput(context.totalSize.get(), stopAt - startAt) - logger.info("End restore,summary: $context $throughput.") - } - - private fun restoreFiles(files: List) { - files.forEach { - val criteria = Criteria.where(ID).isEqualTo(ObjectId(it.id)) - .and(TArchiveFile::status.name).isEqualTo(ArchiveStatus.WAIT_TO_RESTORE.name) - val update = Update().set(TArchiveFile::status.name, ArchiveStatus.RESTORING.name) - .set(TArchiveFile::lastModifiedDate.name, LocalDateTime.now()) - val result = archiveFileDao.updateFirst(Query.query(criteria), update) - if (result.modifiedCount != 1L) { - logger.info("${it.key()} already start restore.") + override fun doOnNext(value: TArchiveFile) { + with(value) { + val tryLock = archiveFileDao.optimisticLock( + value, + TArchiveFile::status.name, + ArchiveStatus.WAIT_TO_RESTORE.name, + ArchiveStatus.RESTORING.name, + ) + if (!tryLock) { + logger.info("$sha256 already start restore.") return } - context.total.incrementAndGet() - context.totalSize.addAndGet(it.size) - try { - logger.info("Start restore file ${it.key()}") - restoreFile(it) - } catch (e: Exception) { - it.status = ArchiveStatus.RESTORE_FAILED - updateArchiveFile(it) - context.failed.incrementAndGet() - logger.error("Restore file ${it.key()} error: ", e) - } - } - } - - private fun restoreFile(file: TArchiveFile) { - with(file) { val key = "$sha256$XZ_SUFFIX" val checkObjectExistRequest = CheckObjectExistRequest(key) val restored = cosClient.checkObjectRestore(checkObjectExistRequest) @@ -129,6 +75,8 @@ class RestoreJob( } val filePath = filesPath.resolve(sha256) try { + logger.info("Start restore file $sha256") + val beginAt = LocalDateTime.now() download(key, filePath) // 存储文件 val artifactFile = filePath.toFile().toArtifactFile(true) @@ -138,13 +86,17 @@ class RestoreJob( } val storageCredentials = ArchiveUtils.getStorageCredentials(storageCredentialsKey) storageService.store(sha256, artifactFile, storageCredentials) - val tp = Throughput(size, Duration.between(lastModifiedDate, LocalDateTime.now()).toNanos()) + val tp = Throughput(size, Duration.between(beginAt, LocalDateTime.now()).toNanos()) val event = FileRestoredEvent(sha256, storageCredentialsKey, tp) SpringContextUtils.publishEvent(event) status = ArchiveStatus.RESTORED updateArchiveFile(this) - context.success.incrementAndGet() logger.info("Success to restore file ${this.key()}.") + } catch (e: Exception) { + value.status = ArchiveStatus.RESTORE_FAILED + updateArchiveFile(value) + logger.error("Restore file $sha256 error: ", e) + throw e } finally { Files.deleteIfExists(filePath) logger.info("Success delete temp file $filePath") @@ -152,6 +104,10 @@ class RestoreJob( } } + override fun getSize(value: TArchiveFile): Long { + return value.size + } + private fun download(key: String, filePath: Path) { val tempFilePath = getTempPath() val xzFilePath = compressedPath.resolve(key) @@ -192,6 +148,6 @@ class RestoreJob( } companion object { - private val logger = LoggerFactory.getLogger(RestoreJob::class.java) + private val logger = LoggerFactory.getLogger(RestoreSubscriber::class.java) } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/CompressJob.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/CompressJob.kt new file mode 100644 index 0000000000..b88c7d752d --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/CompressJob.kt @@ -0,0 +1,49 @@ +package com.tencent.bkrepo.archive.job.compress + +import com.google.common.util.concurrent.ThreadFactoryBuilder +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.config.ArchiveProperties +import com.tencent.bkrepo.archive.model.TCompressFile +import com.tencent.bkrepo.archive.repository.CompressFileDao +import com.tencent.bkrepo.archive.repository.CompressFileRepository +import com.tencent.bkrepo.archive.utils.ArchiveUtils.Companion.newFixedAndCachedThreadPool +import com.tencent.bkrepo.archive.utils.ReactiveDaoUtils +import com.tencent.bkrepo.common.storage.core.StorageService +import java.util.concurrent.TimeUnit +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component +import reactor.core.publisher.Flux +import reactor.core.scheduler.Schedulers + +@Component +class CompressJob( + private val archiveProperties: ArchiveProperties, + private val compressFileDao: CompressFileDao, + private val storageService: StorageService, + private val compressFileRepository: CompressFileRepository, +) { + + private val compressThreadPool = newFixedAndCachedThreadPool( + archiveProperties.ioThreads, + ThreadFactoryBuilder().setNameFormat("storage-compress-%d").build(), + ) + + fun listFiles(): Flux { + val criteria = where(TCompressFile::status).isEqualTo(CompressStatus.CREATED) + val query = Query.query(criteria) + .limit(archiveProperties.queryLimit) + return ReactiveDaoUtils.query(query, TCompressFile::class.java) + } + + @Scheduled(fixedDelay = 1, timeUnit = TimeUnit.HOURS) + fun compress() { + val subscriber = CompressSubscriber(compressFileDao, compressFileRepository, storageService) + listFiles().parallel() + .runOn(Schedulers.fromExecutor(compressThreadPool)) + .subscribe(subscriber) + subscriber.blockLast() + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/CompressSubscriber.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/CompressSubscriber.kt new file mode 100644 index 0000000000..28f520d9a3 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/CompressSubscriber.kt @@ -0,0 +1,82 @@ +package com.tencent.bkrepo.archive.job.compress + +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.event.StorageFileCompressedEvent +import com.tencent.bkrepo.archive.job.BaseJobSubscriber +import com.tencent.bkrepo.archive.model.TCompressFile +import com.tencent.bkrepo.archive.repository.CompressFileDao +import com.tencent.bkrepo.archive.repository.CompressFileRepository +import com.tencent.bkrepo.archive.utils.ArchiveDaoUtils.optimisticLock +import com.tencent.bkrepo.archive.utils.ArchiveUtils +import com.tencent.bkrepo.common.bksync.transfer.exception.TooLowerReuseRateException +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.common.storage.monitor.measureThroughput +import org.slf4j.LoggerFactory +import java.time.LocalDateTime + +class CompressSubscriber( + private val compressFileDao: CompressFileDao, + private val compressFileRepository: CompressFileRepository, + private val storageService: StorageService, +) : BaseJobSubscriber() { + + override fun doOnNext(value: TCompressFile) { + with(value) { + // 乐观锁 + val tryLock = compressFileDao.optimisticLock( + value, + TCompressFile::status.name, + CompressStatus.CREATED.name, + CompressStatus.COMPRESSING.name, + ) + if (!tryLock) { + logger.info("File[$sha256] already start compress.") + return + } + // 压缩 + val credentials = ArchiveUtils.getStorageCredentials(storageCredentialsKey) + var compressedSize = -1L + try { + val throughput = measureThroughput(uncompressedSize) { + compressedSize = storageService.compress(sha256, baseSha256, credentials, true) + } + if (compressedSize == -1L) { + return + } + // 更新状态 + value.compressedSize = compressedSize + value.status = CompressStatus.COMPRESSED + value.lastModifiedDate = LocalDateTime.now() + compressFileRepository.save(value) + val event = StorageFileCompressedEvent( + sha256 = sha256, + baseSha256 = baseSha256, + uncompressed = uncompressedSize, + compressed = compressedSize, + storageCredentialsKey = storageCredentialsKey, + throughput = throughput, + ) + SpringContextUtils.publishEvent(event) + } catch (e: TooLowerReuseRateException) { + logger.info("Reuse rate is too lower.") + value.status = CompressStatus.COMPRESS_FAILED + value.lastModifiedDate = LocalDateTime.now() + compressFileRepository.save(value) + } catch (e: Exception) { + value.status = CompressStatus.COMPRESS_FAILED + value.lastModifiedDate = LocalDateTime.now() + compressFileRepository.save(value) + throw e + } + } + } + + override fun getSize(value: TCompressFile): Long { + return value.uncompressedSize + } + + companion object { + private val logger = LoggerFactory.getLogger(CompressSubscriber::class.java) + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/UncompressJob.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/UncompressJob.kt new file mode 100644 index 0000000000..4564ac68a8 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/UncompressJob.kt @@ -0,0 +1,49 @@ +package com.tencent.bkrepo.archive.job.compress + +import com.google.common.util.concurrent.ThreadFactoryBuilder +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.config.ArchiveProperties +import com.tencent.bkrepo.archive.model.TCompressFile +import com.tencent.bkrepo.archive.repository.CompressFileDao +import com.tencent.bkrepo.archive.repository.CompressFileRepository +import com.tencent.bkrepo.archive.utils.ArchiveUtils +import com.tencent.bkrepo.archive.utils.ReactiveDaoUtils +import com.tencent.bkrepo.common.storage.core.StorageService +import java.util.concurrent.TimeUnit +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component +import reactor.core.publisher.Flux +import reactor.core.scheduler.Schedulers + +@Component +class UncompressJob( + private val archiveProperties: ArchiveProperties, + private val compressFileDao: CompressFileDao, + private val storageService: StorageService, + private val compressFileRepository: CompressFileRepository, +) { + + private val uncompressThreadPool = ArchiveUtils.newFixedAndCachedThreadPool( + archiveProperties.ioThreads, + ThreadFactoryBuilder().setNameFormat("storage-uncompress-%d").build(), + ) + + fun listFiles(): Flux { + val criteria = where(TCompressFile::status).isEqualTo(CompressStatus.WAIT_TO_UNCOMPRESS) + val query = Query.query(criteria) + .limit(archiveProperties.queryLimit) + return ReactiveDaoUtils.query(query, TCompressFile::class.java) + } + + @Scheduled(fixedDelay = 1, timeUnit = TimeUnit.HOURS) + fun uncompress() { + val subscriber = UncompressSubscriber(compressFileDao, compressFileRepository, storageService) + listFiles().parallel() + .runOn(Schedulers.fromExecutor(uncompressThreadPool)) + .subscribe(subscriber) + subscriber.blockLast() + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/UncompressSubscriber.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/UncompressSubscriber.kt new file mode 100644 index 0000000000..26ab379933 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/job/compress/UncompressSubscriber.kt @@ -0,0 +1,67 @@ +package com.tencent.bkrepo.archive.job.compress + +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.event.StorageFileUncompressedEvent +import com.tencent.bkrepo.archive.job.BaseJobSubscriber +import com.tencent.bkrepo.archive.model.TCompressFile +import com.tencent.bkrepo.archive.repository.CompressFileDao +import com.tencent.bkrepo.archive.repository.CompressFileRepository +import com.tencent.bkrepo.archive.utils.ArchiveDaoUtils.optimisticLock +import com.tencent.bkrepo.archive.utils.ArchiveUtils +import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.common.storage.monitor.measureThroughput +import java.time.LocalDateTime +import org.slf4j.LoggerFactory + +class UncompressSubscriber( + private val compressFileDao: CompressFileDao, + private val compressFileRepository: CompressFileRepository, + private val storageService: StorageService, +) : BaseJobSubscriber() { + + override fun doOnNext(value: TCompressFile) { + with(value) { + // 乐观锁 + val tryLock = compressFileDao.optimisticLock( + value, + TCompressFile::status.name, + CompressStatus.WAIT_TO_UNCOMPRESS.name, + CompressStatus.UNCOMPRESSING.name, + ) + if (!tryLock) { + logger.info("File[$sha256] already start uncompress.") + } + // 解压 + val credentials = ArchiveUtils.getStorageCredentials(storageCredentialsKey) + try { + val throughput = measureThroughput(uncompressedSize) { storageService.uncompress(sha256, credentials) } + // 更新状态 + value.status = CompressStatus.UNCOMPRESSED + value.lastModifiedDate = LocalDateTime.now() + compressFileRepository.save(value) + val event = StorageFileUncompressedEvent( + sha256 = sha256, + compressed = compressedSize, + uncompressed = uncompressedSize, + storageCredentialsKey = storageCredentialsKey, + throughput = throughput, + ) + SpringContextUtils.publishEvent(event) + } catch (e: Exception) { + value.status = CompressStatus.UNCOMPRESS_FAILED + value.lastModifiedDate = LocalDateTime.now() + compressFileRepository.save(value) + throw e + } + } + } + + override fun getSize(value: TCompressFile): Long { + return value.uncompressedSize + } + + companion object { + private val logger = LoggerFactory.getLogger(UncompressSubscriber::class.java) + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/listener/ArchiveListener.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/listener/ArchiveListener.kt index 32094a6bb9..0ed9e515b3 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/listener/ArchiveListener.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/listener/ArchiveListener.kt @@ -1,10 +1,11 @@ package com.tencent.bkrepo.archive.listener +import com.tencent.bkrepo.archive.constant.DEFAULT_KEY import com.tencent.bkrepo.archive.event.FileArchivedEvent import com.tencent.bkrepo.archive.event.FileCompressedEvent import com.tencent.bkrepo.archive.event.FileRestoredEvent import com.tencent.bkrepo.archive.metrics.ArchiveMetrics -import java.text.DecimalFormat +import com.tencent.bkrepo.common.api.constant.StringPool import org.slf4j.LoggerFactory import org.springframework.context.event.EventListener import org.springframework.stereotype.Component @@ -16,10 +17,8 @@ import org.springframework.stereotype.Component class ArchiveListener(val archiveMetrics: ArchiveMetrics) { /** - * 压缩比格式 + * 归档文件事件 * */ - private val df = DecimalFormat("#.#") - @EventListener(FileArchivedEvent::class) fun archive(event: FileArchivedEvent) { with(event) { @@ -31,6 +30,9 @@ class ArchiveListener(val archiveMetrics: ArchiveMetrics) { } } + /** + * 归档文件恢复事件 + * */ @EventListener(FileRestoredEvent::class) fun restore(event: FileRestoredEvent) { with(event) { @@ -42,21 +44,21 @@ class ArchiveListener(val archiveMetrics: ArchiveMetrics) { } } + /** + * 归档文件压缩事件 + * */ @EventListener(FileCompressedEvent::class) fun compress(event: FileCompressedEvent) { with(event) { - val ratio = df.format((uncompressed - compressed.toDouble()) / uncompressed * 100) - logger.info("Compress file $sha256, compressed:$compressed,uncompressed:$uncompressed,ratio:$ratio") - archiveMetrics.getCompressSizeCount(ArchiveMetrics.CompressCounterType.COMPRESSED.name) - .increment(compressed.toDouble()) - archiveMetrics.getCompressSizeCount(ArchiveMetrics.CompressCounterType.UNCOMPRESSED.name) - .increment(uncompressed.toDouble()) - archiveMetrics.getCompressTimer().record(throughput.duration) + val ratio = StringPool.calculateRatio(uncompressed, compressed) + logger.info( + "Archive utils compress file $sha256,compressed:$compressed," + + "uncompressed:$uncompressed,ratio:$ratio, $throughput", + ) } } companion object { private val logger = LoggerFactory.getLogger(ArchiveListener::class.java) - private const val DEFAULT_KEY = "default" } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/listener/StorageCompressListener.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/listener/StorageCompressListener.kt new file mode 100644 index 0000000000..42b5126ca6 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/listener/StorageCompressListener.kt @@ -0,0 +1,65 @@ +package com.tencent.bkrepo.archive.listener + +import com.tencent.bkrepo.archive.constant.DEFAULT_KEY +import com.tencent.bkrepo.archive.event.StorageFileCompressedEvent +import com.tencent.bkrepo.archive.event.StorageFileUncompressedEvent +import com.tencent.bkrepo.archive.metrics.ArchiveMetrics +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.util.HumanReadable +import org.slf4j.LoggerFactory +import org.springframework.context.event.EventListener +import org.springframework.stereotype.Component + +/** + * 存储压缩事件监听器 + * */ +@Component +class StorageCompressListener(val archiveMetrics: ArchiveMetrics) { + /** + * 压缩存储文件 + * */ + @EventListener(StorageFileCompressedEvent::class) + fun compress(event: StorageFileCompressedEvent) { + with(event) { + val ratio = StringPool.calculateRatio(uncompressed, compressed) + val uncompressedSize = HumanReadable.size(uncompressed) + val compressedSize = HumanReadable.size(compressed) + val releaseSize = HumanReadable.size(uncompressed - compressed) + logger.info( + "Success to compress file $sha256 on $storageCredentialsKey," + + "($uncompressedSize->$compressedSize,$releaseSize) ratio:$ratio ,$throughput", + ) + val key = event.storageCredentialsKey ?: DEFAULT_KEY + archiveMetrics.getCounter(ArchiveMetrics.Action.COMPRESSED, key).increment() + archiveMetrics.getSizeCounter(ArchiveMetrics.Action.COMPRESSED, key, TYPE, TAG_COMPRESSED) + .increment(compressed.toDouble()) + archiveMetrics.getSizeCounter(ArchiveMetrics.Action.COMPRESSED, key, TYPE, TAG_UNCOMPRESSED) + .increment(uncompressed.toDouble()) + archiveMetrics.getTimer(ArchiveMetrics.Action.COMPRESSED, key).record(throughput.duration) + } + } + + /** + * 解压存储文件 + * */ + @EventListener(StorageFileUncompressedEvent::class) + fun uncompress(event: StorageFileUncompressedEvent) { + with(event) { + logger.info("Success to uncompress file $sha256 on $storageCredentialsKey,$throughput") + val key = event.storageCredentialsKey ?: DEFAULT_KEY + archiveMetrics.getCounter(ArchiveMetrics.Action.UNCOMPRESSED, key).increment() + archiveMetrics.getSizeCounter(ArchiveMetrics.Action.UNCOMPRESSED, key, TYPE, TAG_COMPRESSED) + .increment(compressed.toDouble()) + archiveMetrics.getSizeCounter(ArchiveMetrics.Action.UNCOMPRESSED, key, TYPE, TAG_UNCOMPRESSED) + .increment(uncompressed.toDouble()) + archiveMetrics.getTimer(ArchiveMetrics.Action.UNCOMPRESSED, key).record(throughput.duration) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(StorageCompressListener::class.java) + private const val TYPE = "type" + private const val TAG_COMPRESSED = "compressed" + private const val TAG_UNCOMPRESSED = "uncompressed" + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/metrics/ArchiveMetrics.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/metrics/ArchiveMetrics.kt index 063d3edd19..378c31ab1b 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/metrics/ArchiveMetrics.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/metrics/ArchiveMetrics.kt @@ -1,8 +1,10 @@ package com.tencent.bkrepo.archive.metrics import com.tencent.bkrepo.archive.ArchiveStatus -import com.tencent.bkrepo.archive.job.ReactiveArchiveJob +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.job.archive.ArchiveJob import com.tencent.bkrepo.archive.repository.ArchiveFileRepository +import com.tencent.bkrepo.archive.repository.CompressFileRepository import io.micrometer.core.instrument.Counter import io.micrometer.core.instrument.Gauge import io.micrometer.core.instrument.MeterRegistry @@ -10,12 +12,17 @@ import io.micrometer.core.instrument.Timer import io.micrometer.core.instrument.binder.MeterBinder import org.springframework.stereotype.Component +/** + * 归档度量指标 + * */ @Component class ArchiveMetrics( - val archiveJob: ReactiveArchiveJob, + val archiveJob: ArchiveJob, val archiveFileRepository: ArchiveFileRepository, + val compressFileRepository: CompressFileRepository, ) : MeterBinder { lateinit var registry: MeterRegistry + override fun bindTo(registry: MeterRegistry) { this.registry = registry // 下载量，队列 @@ -44,76 +51,103 @@ class ArchiveMetrics( // 归档文件状态 ArchiveStatus.values().forEach { - Gauge.builder(FILE_STATUS_COUNTER) { archiveFileRepository.countByStatus(it) } - .description(FILE_STATUS_COUNTER_DESC) + Gauge.builder(ARCHIVE_FILE_STATUS_COUNTER) { archiveFileRepository.countByStatus(it) } + .description(ARCHIVE_FILE_STATUS_COUNTER_DESC) + .tag(TAG_STATUS, it.name) + .register(registry) + } + + // 压缩文件状态 + CompressStatus.values().forEach { + Gauge.builder(COMPRESS_FILE_STATUS_COUNTER) { compressFileRepository.countByStatus(it) } + .description(COMPRESS_FILE_STATUS_COUNTER_DESC) .tag(TAG_STATUS, it.name) .register(registry) } } /** - * 数量 + * 获取数量计数器 + * @param action 当前操作 + * @param credentialsKey 存储key * */ - fun getCounter(action: Action, key: String): Counter { - val c = if (action == Action.ARCHIVED) { - Counter.builder(FILE_ARCHIVED_COUNTER) + fun getCounter(action: Action, credentialsKey: String): Counter { + val builder = when (action) { + Action.ARCHIVED -> Counter.builder(FILE_ARCHIVED_COUNTER) .description(FILE_ARCHIVED_COUNTER_DESC) - } else { - Counter.builder(FILE_RESTORED_COUNTER) + + Action.RESTORED -> Counter.builder(FILE_RESTORED_COUNTER) .description(FILE_RESTORED_COUNTER_DESC) + + Action.COMPRESSED -> Counter.builder(FILE_COMPRESSED_COUNTER) + .description(FILE_COMPRESSED_COUNTER_DESC) + + Action.UNCOMPRESSED -> Counter.builder(FILE_UNCOMPRESSED_COUNTER) + .description(FILE_UNCOMPRESSED_COUNTER_DESC) } - return c.tag(TAG_CREDENTIALS_KEY, key) + return builder.tag(TAG_CREDENTIALS_KEY, credentialsKey) .register(registry) } /** - * 大小 + * 获取大小计数器 + * @param action 当前操作 + * @param credentialsKey 存储key + * @param tags 额外tag * */ - fun getSizeCounter(action: Action, key: String): Counter { - val c = if (action == Action.ARCHIVED) { - Counter.builder(FILE_ARCHIVED_SIZE_COUNTER) + fun getSizeCounter(action: Action, credentialsKey: String, vararg tags: String): Counter { + val builder = when (action) { + Action.ARCHIVED -> Counter.builder(FILE_ARCHIVED_SIZE_COUNTER) .description(FILE_ARCHIVED_SIZE_COUNTER_DESC) - } else { - Counter.builder(FILE_RESTORED_SIZE_COUNTER) + + Action.RESTORED -> Counter.builder(FILE_RESTORED_SIZE_COUNTER) .description(FILE_RESTORED_SIZE_COUNTER_DESC) + + Action.COMPRESSED -> Counter.builder(FILE_COMPRESSED_SIZE_COUNTER) + .description(FILE_COMPRESSED_SIZE_COUNTER_DESC) + + Action.UNCOMPRESSED -> Counter.builder(FILE_UNCOMPRESSED_SIZE_COUNTER) + .description(FILE_UNCOMPRESSED_SIZE_COUNTER_DESC) } - return c.tag(TAG_CREDENTIALS_KEY, key) + require(tags.size % 2 == 0) + for (i in 0 until tags.lastIndex) { + val key = tags[i] + val value = tags[i + 1] + builder.tag(key, value) + } + return builder.tag(TAG_CREDENTIALS_KEY, credentialsKey) .register(registry) } /** - * 计时器 + * 获取计时器 + * @param action 当前操作 + * @param credentialsKey 存储key * */ - fun getTimer(action: Action, key: String): Timer { - val c = if (action == Action.ARCHIVED) { - Timer.builder(FILE_ARCHIVED_TIME) + fun getTimer(action: Action, credentialsKey: String): Timer { + val builder = when (action) { + Action.ARCHIVED -> Timer.builder(FILE_ARCHIVED_TIME) .description(FILE_ARCHIVED_TIME_DESC) - } else { - Timer.builder(FILE_RESTORED_TIME) + + Action.RESTORED -> Timer.builder(FILE_RESTORED_TIME) .description(FILE_RESTORED_TIME_DESC) - } - return c.tag(TAG_CREDENTIALS_KEY, key).register(registry) - } - fun getCompressSizeCount(type: String): Counter { - return Counter.builder(FILE_COMPRESS_SIZE_COUNTER) - .description(FILE_COMPRESS_SIZE_COUNTER_DESC) - .tag(TAG_TYPE, type) - .register(registry) - } + Action.COMPRESSED -> Timer.builder(FILE_COMPRESSED_TIME) + .description(FILE_COMPRESSED_TIME_DESC) - fun getCompressTimer(): Timer { - return Timer.builder(FILE_COMPRESS_TIME) - .description(FILE_COMPRESS_TIME_DESC) + Action.UNCOMPRESSED -> Timer.builder(FILE_UNCOMPRESSED_TIME) + .description(FILE_UNCOMPRESSED_TIME_DESC) + } + return builder.tag(TAG_CREDENTIALS_KEY, credentialsKey) .register(registry) } + /** + * 归档服务相关动作 + * */ enum class Action { ARCHIVED, RESTORED, - } - - enum class CompressCounterType { COMPRESSED, UNCOMPRESSED, } @@ -143,14 +177,23 @@ class ArchiveMetrics( private const val FILE_COMPRESS_ACTIVE_COUNT_DESC = "文件压缩实时数量" private const val FILE_COMPRESS_QUEUE_SIZE = "file.compress.queue.size" private const val FILE_COMPRESS_QUEUE_SIZE_DESC = "文件压缩队列大小" - private const val FILE_STATUS_COUNTER = "file.status.count" - private const val FILE_STATUS_COUNTER_DESC = "文件状态统计" - private const val FILE_COMPRESS_SIZE_COUNTER = "file.compress.size.count" - private const val FILE_COMPRESS_SIZE_COUNTER_DESC = "文件压缩大小" - private const val FILE_COMPRESS_TIME = "file.compress.time" - private const val FILE_COMPRESS_TIME_DESC = "文件压缩耗时" + private const val ARCHIVE_FILE_STATUS_COUNTER = "file.archive.status.count" + private const val ARCHIVE_FILE_STATUS_COUNTER_DESC = "归档文件状态统计" + private const val COMPRESS_FILE_STATUS_COUNTER = "file.compress.status.count" + private const val COMPRESS_FILE_STATUS_COUNTER_DESC = "压缩文件状态统计" + private const val FILE_COMPRESSED_COUNTER = "file.compress.count" + private const val FILE_COMPRESSED_COUNTER_DESC = "文件压缩数量" + private const val FILE_COMPRESSED_SIZE_COUNTER = "file.compress.size.count" + private const val FILE_COMPRESSED_SIZE_COUNTER_DESC = "文件压缩大小" + private const val FILE_COMPRESSED_TIME = "file.compress.time" + private const val FILE_COMPRESSED_TIME_DESC = "文件压缩耗时" + private const val FILE_UNCOMPRESSED_COUNTER = "file.uncompress.count" + private const val FILE_UNCOMPRESSED_COUNTER_DESC = "文件压缩数量" + private const val FILE_UNCOMPRESSED_SIZE_COUNTER = "file.uncompress.size.count" + private const val FILE_UNCOMPRESSED_SIZE_COUNTER_DESC = "文件解压大小" + private const val FILE_UNCOMPRESSED_TIME = "file.uncompress.time" + private const val FILE_UNCOMPRESSED_TIME_DESC = "文件解压耗时" private const val TAG_CREDENTIALS_KEY = "credentialsKey" private const val TAG_STATUS = "status" - private const val TAG_TYPE = "type" } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/AbstractEntity.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/AbstractEntity.kt new file mode 100644 index 0000000000..8562af7a19 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/AbstractEntity.kt @@ -0,0 +1,14 @@ +package com.tencent.bkrepo.archive.model + +import java.time.LocalDateTime + +/** + * 抽象实体类 + * */ +open class AbstractEntity( + id: String? = null, + val createdBy: String, + val createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, +) : IdEntity(id) diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/IdEntity.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/IdEntity.kt new file mode 100644 index 0000000000..bf36a8eb0f --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/IdEntity.kt @@ -0,0 +1,8 @@ +package com.tencent.bkrepo.archive.model + +/** + * id entity + * */ +open class IdEntity( + var id: String? = null, +) diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/TArchiveFile.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/TArchiveFile.kt index ce3469dee7..90fb635476 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/TArchiveFile.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/TArchiveFile.kt @@ -15,16 +15,23 @@ import org.springframework.data.mongodb.core.mapping.Document CompoundIndex(name = SHA256_IDX, def = SHA256_IDX_DEF, unique = true, background = true), CompoundIndex(name = STATUS_IDX, def = STATUS_IDX_DEF, background = true), ) -data class TArchiveFile( - var id: String? = null, - var createdBy: String, - var createdDate: LocalDateTime, - var lastModifiedBy: String, - var lastModifiedDate: LocalDateTime, +@Suppress("LongParameterList") +class TArchiveFile( + id: String? = null, + createdBy: String, + createdDate: LocalDateTime, + lastModifiedBy: String, + lastModifiedDate: LocalDateTime, val sha256: String, val size: Long, val storageCredentialsKey: String?, var status: ArchiveStatus, +) : AbstractEntity( + id, + createdBy, + createdDate, + lastModifiedBy, + lastModifiedDate, ) { companion object { const val SHA256_IDX = "sha256_storageCredentialsKey_idx" diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/TCompressFile.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/TCompressFile.kt new file mode 100644 index 0000000000..c371310989 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/model/TCompressFile.kt @@ -0,0 +1,45 @@ +package com.tencent.bkrepo.archive.model + +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.model.TCompressFile.Companion.SHA256_IDX +import com.tencent.bkrepo.archive.model.TCompressFile.Companion.SHA256_IDX_DEF +import com.tencent.bkrepo.archive.model.TCompressFile.Companion.STATUS_IDX +import com.tencent.bkrepo.archive.model.TCompressFile.Companion.STATUS_IDX_DEF +import java.time.LocalDateTime +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document + +@Document("compress_file") +@CompoundIndexes( + CompoundIndex(name = SHA256_IDX, def = SHA256_IDX_DEF, unique = true, background = true), + CompoundIndex(name = STATUS_IDX, def = STATUS_IDX_DEF, background = true), +) +@Suppress("LongParameterList") +class TCompressFile( + id: String? = null, + createdBy: String, + createdDate: LocalDateTime, + lastModifiedBy: String, + lastModifiedDate: LocalDateTime, + val sha256: String, + val baseSha256: String, + val uncompressedSize: Long, + var compressedSize: Long = -1, + val storageCredentialsKey: String?, + var status: CompressStatus, + var chainLength: Int = -1, // 只有队头元素有 +) : AbstractEntity( + id, + createdBy, + createdDate, + lastModifiedBy, + lastModifiedDate, +) { + companion object { + const val SHA256_IDX = "sha256_storageCredentialsKey_idx" + const val SHA256_IDX_DEF = "{'sha256': 1,'storageCredentialsKey': 1}" + const val STATUS_IDX = "status_idx" + const val STATUS_IDX_DEF = "{'status': 1}" + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/repository/CompressFileDao.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/repository/CompressFileDao.kt new file mode 100644 index 0000000000..926849f15e --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/repository/CompressFileDao.kt @@ -0,0 +1,8 @@ +package com.tencent.bkrepo.archive.repository + +import com.tencent.bkrepo.archive.model.TCompressFile +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import org.springframework.stereotype.Repository + +@Repository +class CompressFileDao : SimpleMongoDao() diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/repository/CompressFileRepository.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/repository/CompressFileRepository.kt new file mode 100644 index 0000000000..df37e1f28e --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/repository/CompressFileRepository.kt @@ -0,0 +1,12 @@ +package com.tencent.bkrepo.archive.repository + +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.model.TCompressFile +import org.springframework.data.mongodb.repository.MongoRepository +import org.springframework.stereotype.Repository + +@Repository +interface CompressFileRepository : MongoRepository { + fun findBySha256AndStorageCredentialsKey(sha256: String, key: String?): TCompressFile? + fun countByStatus(status: CompressStatus): Int +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt index 01cdbb536f..1db89d4c51 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/ArchiveServiceImpl.kt @@ -18,6 +18,9 @@ import java.time.LocalDateTime import org.slf4j.LoggerFactory import org.springframework.stereotype.Service +/** + * 归档服务实现类 + * */ @Service class ArchiveServiceImpl( private val archiveProperties: ArchiveProperties, @@ -29,12 +32,15 @@ class ArchiveServiceImpl( // created with(request) { val af = archiveFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey) - // 允许再次归档 if (af != null) { - // update status to created + // 文件已归档 + if (af.status == ArchiveStatus.ARCHIVED) { + return + } af.lastModifiedBy = operator af.lastModifiedDate = LocalDateTime.now() - af.status = ArchiveStatus.CREATED + // 重新触发归档后逻辑，删除原存储文件和更新node状态 + af.status = if (af.status == ArchiveStatus.COMPLETED) ArchiveStatus.ARCHIVED else ArchiveStatus.CREATED archiveFileRepository.save(af) } else { val archiveFile = TArchiveFile( diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt new file mode 100644 index 0000000000..fb682b6b34 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressService.kt @@ -0,0 +1,33 @@ +package com.tencent.bkrepo.archive.service + +import com.tencent.bkrepo.archive.request.CompleteCompressRequest +import com.tencent.bkrepo.archive.request.CompressFileRequest +import com.tencent.bkrepo.archive.request.DeleteCompressRequest +import com.tencent.bkrepo.archive.request.UncompressFileRequest + +/** + * 压缩服务 + * */ +interface CompressService { + /** + * 压缩文件 + * */ + fun compress(request: CompressFileRequest) + + /** + * 解压文件 + * */ + + fun uncompress(request: UncompressFileRequest) + + /** + * 删除压缩文件 + * */ + + fun delete(request: DeleteCompressRequest) + + /** + * 完成压缩 + * */ + fun complete(request: CompleteCompressRequest) +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt new file mode 100644 index 0000000000..bf930b324e --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt @@ -0,0 +1,150 @@ +package com.tencent.bkrepo.archive.service + +import com.tencent.bkrepo.archive.ArchiveFileNotFound +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.model.TCompressFile +import com.tencent.bkrepo.archive.repository.CompressFileRepository +import com.tencent.bkrepo.archive.request.CompleteCompressRequest +import com.tencent.bkrepo.archive.request.CompressFileRequest +import com.tencent.bkrepo.archive.request.DeleteCompressRequest +import com.tencent.bkrepo.archive.request.UncompressFileRequest +import com.tencent.bkrepo.archive.utils.ArchiveUtils +import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.repository.api.FileReferenceClient +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import java.time.LocalDateTime + +/** + * 压缩服务实现类 + * */ +@Service +class CompressServiceImpl( + private val compressFileRepository: CompressFileRepository, + private val storageService: StorageService, + private val fileReferenceClient: FileReferenceClient, +) : CompressService { + + override fun compress(request: CompressFileRequest) { + with(request) { + // 队头元素 + val head = compressFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey) + if (head != null && head.status != CompressStatus.NONE) { + // 压缩任务已存在 + logger.info("Compress file [$sha256] already exists，status: ${head.status}.") + head.lastModifiedBy = operator + head.lastModifiedDate = LocalDateTime.now() + // 重新触发压缩后逻辑，删除原存储文件和更新node状态 + head.status = if (head.status == CompressStatus.COMPLETED) { + CompressStatus.COMPRESSED + } else { + CompressStatus.CREATED + } + return + } + var currentChainLength = 0 + // 这是队头 + if (head != null) { + currentChainLength = head.chainLength + 1 + // 超出链最大长度限制 + if (currentChainLength > MAX_CHAIN_LENGTH) { + // 超出队列长度 + logger.info("Exceed max chain length,ignore it.") + return + } + // 删除旧头 + compressFileRepository.delete(head) + } + val compressFile = TCompressFile( + createdBy = operator, + createdDate = LocalDateTime.now(), + lastModifiedBy = operator, + lastModifiedDate = LocalDateTime.now(), + sha256 = sha256, + baseSha256 = baseSha256, + uncompressedSize = size, + storageCredentialsKey = storageCredentialsKey, + status = CompressStatus.CREATED, + ) + compressFileRepository.save(compressFile) + val newChain = compressFileRepository.findBySha256AndStorageCredentialsKey( + baseSha256, + storageCredentialsKey, + ) ?: TCompressFile( + createdBy = operator, + createdDate = LocalDateTime.now(), + lastModifiedBy = operator, + lastModifiedDate = LocalDateTime.now(), + sha256 = baseSha256, + baseSha256 = "", + uncompressedSize = size, + storageCredentialsKey = storageCredentialsKey, + status = CompressStatus.NONE, + chainLength = 1, + ) + /* + * 确定新链长度，取最长链长度 + * 1. sha256链长度+1 + * 2. baseSha256所在链长 + * */ + val newChainLength = maxOf(newChain.chainLength, currentChainLength) + newChain.chainLength = newChainLength + compressFileRepository.save(newChain) + fileReferenceClient.increment(baseSha256, storageCredentialsKey) + logger.info("Compress file [$sha256] on $storageCredentialsKey.") + } + } + + override fun uncompress(request: UncompressFileRequest) { + with(request) { + val file = compressFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey) + ?: throw ArchiveFileNotFound(sha256) + if (file.status == CompressStatus.COMPLETED || file.status == CompressStatus.COMPRESSED) { + file.status = CompressStatus.WAIT_TO_UNCOMPRESS + file.lastModifiedBy = operator + file.lastModifiedDate = LocalDateTime.now() + compressFileRepository.save(file) + logger.info("Uncompress file [$sha256] on $storageCredentialsKey.") + } + } + } + + override fun delete(request: DeleteCompressRequest) { + with(request) { + val file = compressFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey) + ?: return + if (file.status != CompressStatus.NONE) { + val storageCredentials = ArchiveUtils.getStorageCredentials(storageCredentialsKey) + if (storageService.isCompressed(sha256, storageCredentials)) { + storageService.deleteCompressed(sha256, storageCredentials) + } + fileReferenceClient.decrement(file.baseSha256, storageCredentialsKey) + /* + * 解压是小概率事件，所以这里链长度我们就不减少，这样带来的问题是， + * 压缩链更容易达到最大长度。但是这个影响并不重要。 + * */ + compressFileRepository.delete(file) + logger.info("Delete compress file [$sha256].") + } + } + } + + override fun complete(request: CompleteCompressRequest) { + with(request) { + val file = compressFileRepository.findBySha256AndStorageCredentialsKey(sha256, storageCredentialsKey) + ?: throw ArchiveFileNotFound(sha256) + if (file.status == CompressStatus.COMPRESSED) { + file.status = CompressStatus.COMPLETED + file.lastModifiedBy = operator + file.lastModifiedDate = LocalDateTime.now() + compressFileRepository.save(file) + logger.info("Complete compress file [$sha256].") + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(CompressServiceImpl::class.java) + private const val MAX_CHAIN_LENGTH = 10 + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveDaoUtils.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveDaoUtils.kt new file mode 100644 index 0000000000..4e0f40f57f --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveDaoUtils.kt @@ -0,0 +1,37 @@ +package com.tencent.bkrepo.archive.utils + +import com.tencent.bkrepo.archive.model.AbstractEntity +import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.mongo.dao.AbstractMongoDao +import org.bson.types.ObjectId +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import java.time.LocalDateTime + +/** + * 归档DAO工具 + * */ +object ArchiveDaoUtils { + /** + * 乐观锁 + * 通过cap实现乐观锁 + * @param entity 操作实体 + * @param key 需要原子更新的key + * @param expect 期望值 + * @param update 更改值 + * @return true更改成功，false更改失败 + * */ + fun AbstractMongoDao.optimisticLock( + entity: E, + key: String, + expect: Any, + update: Any, + ): Boolean { + val criteria = Criteria.where(ID).isEqualTo(ObjectId(entity.id)).and(key).isEqualTo(expect) + val newUpdate = Update().set(key, update) + .set(entity::lastModifiedDate.name, LocalDateTime.now()) + return this.updateFirst(Query.query(criteria), newUpdate).modifiedCount == 1L + } +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveUtils.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveUtils.kt index d71d6cc2f3..cdb4cb9e1e 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveUtils.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ArchiveUtils.kt @@ -4,9 +4,15 @@ import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader import com.google.common.cache.LoadingCache import com.tencent.bkrepo.archive.job.JobProcessMonitor +import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient +import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.ThreadFactory +import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit import org.slf4j.LoggerFactory import org.springframework.stereotype.Component @@ -15,21 +21,28 @@ import org.springframework.stereotype.Component class ArchiveUtils( storageCredentialsClient: StorageCredentialsClient, storageProperties: StorageProperties, + repositoryClient: RepositoryClient, ) { init { Companion.storageCredentialsClient = storageCredentialsClient + Companion.repositoryClient = repositoryClient defaultStorageCredentials = storageProperties.defaultStorageCredentials() } companion object { private lateinit var storageCredentialsClient: StorageCredentialsClient private lateinit var defaultStorageCredentials: StorageCredentials + private lateinit var repositoryClient: RepositoryClient private val logger = LoggerFactory.getLogger(ArchiveUtils::class.java) private val storageCredentialsCache: LoadingCache = CacheBuilder.newBuilder() .maximumSize(1000) .expireAfterWrite(1, TimeUnit.MINUTES) .build(CacheLoader.from { key -> loadStorageCredentials(key) }) + private val repositoryDetailCache = CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite(60, TimeUnit.SECONDS) + .build() val monitor = JobProcessMonitor() @@ -46,6 +59,14 @@ class ArchiveUtils( return storageCredentialsCache.get(key.orEmpty()) } + fun getRepositoryDetail(project: String, repoName: String): RepositoryDetail { + val repoId = RepositoryId(project, repoName) + return repositoryDetailCache.get(repoId) { + repositoryClient.getRepoDetail(project, repoName).data + ?: throw RepoNotFoundException("$project/$repoName") + } + } + fun supportXZCmd(): Boolean { return try { Runtime.getRuntime().exec("xz -V") @@ -55,6 +76,18 @@ class ArchiveUtils( } } + fun newFixedAndCachedThreadPool(threads: Int, threadFactory: ThreadFactory): ThreadPoolExecutor { + return ThreadPoolExecutor( + threads, + threads, + 60, + TimeUnit.SECONDS, + ArrayBlockingQueue(DEFAULT_BUFFER_SIZE), + threadFactory, + ThreadPoolExecutor.CallerRunsPolicy(), + ) + } + fun runCmd(cmd: List) { logger.debug("# ${cmd.joinToString(" ")}") val pb = ProcessBuilder() @@ -65,5 +98,10 @@ class ArchiveUtils( error("cmd failed,exit: $ev.") } } + + data class RepositoryId( + val project: String, + val repoName: String, + ) } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ReactiveDaoUtils.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ReactiveDaoUtils.kt new file mode 100644 index 0000000000..267e14f3e0 --- /dev/null +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/utils/ReactiveDaoUtils.kt @@ -0,0 +1,66 @@ +package com.tencent.bkrepo.archive.utils + +import com.tencent.bkrepo.archive.model.IdEntity +import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID +import org.bson.types.ObjectId +import org.reactivestreams.Publisher +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.ReactiveMongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.stereotype.Component +import reactor.core.publisher.Flux +import reactor.core.publisher.FluxSink +import reactor.core.publisher.Mono + +@Component +class ReactiveDaoUtils(reactiveMongoTemplate: ReactiveMongoTemplate) { + init { + Companion.reactiveMongoTemplate = reactiveMongoTemplate + } + + companion object { + lateinit var reactiveMongoTemplate: ReactiveMongoTemplate + fun query(query: Query, clazz: Class): Flux { + val idQueryCursor = IdQueryCursor(query, reactiveMongoTemplate, clazz) + return Flux.create { recurseCursor(idQueryCursor, it) } + } + + private fun recurseCursor(idQueryCursor: IdQueryCursor, sink: FluxSink) { + Mono.from(idQueryCursor.next()) + .doOnSuccess { results -> + results.forEach { sink.next(it) } + if (idQueryCursor.hasNext) { + recurseCursor(idQueryCursor, sink) + } else { + sink.complete() + } + }.subscribe() + } + } + + /** + * id查询游标，将查询使用id在进行分页查找，避免大表skip，导致性能下降 + * */ + private class IdQueryCursor( + val query: Query, + val reactiveMongoTemplate: ReactiveMongoTemplate, + val clazz: Class, + ) { + private var lastId = MIN_OBJECT_ID + var hasNext: Boolean = true + + fun next(): Publisher> { + val idQuery = Query.of(query).addCriteria(Criteria.where(ID).gt(ObjectId(lastId))) + .with(Sort.by(ID).ascending()) + return reactiveMongoTemplate.find(idQuery, clazz).collectList() + .doOnSuccess { + if (it.isNotEmpty()) { + lastId = it.last().id!! + } + hasNext = it.size == query.limit + } + } + } +} diff --git a/src/backend/common/common-api/build.gradle.kts b/src/backend/common/common-api/build.gradle.kts index aa123dfe8b..31f47ab4d6 100644 --- a/src/backend/common/common-api/build.gradle.kts +++ b/src/backend/common/common-api/build.gradle.kts @@ -39,4 +39,5 @@ dependencies { api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml") api("com.fasterxml.jackson.module:jackson-module-parameter-names") api("org.apache.commons:commons-compress") + api("com.google.guava:guava") } diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt new file mode 100644 index 0000000000..3aed90d413 --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/collection/CollectionExtensions.kt @@ -0,0 +1,43 @@ +package com.tencent.bkrepo.common.api.collection + +/** + * 支持大数据集的聚合，元素会优先加入相似的组，而不是最相似的组。 + * 因为该聚合算法最后的分组不是一个最佳的聚合方式，这是对时间复杂度和数据准确度的一个折中。 + * 算法思路： + * 1. 先对key进行排序（排序的复杂度远小于聚合，同时我们认为key排序接近的，也是应该是相似的） + * 2. 遍历每一项。我们使用三个分组来处理数据，一个新数据，要么加入当前组，要么加入之前的分组（为了防止噪点），要么自己新成立一个组。 + * 3. 分组完成 （这里如果是最佳聚合，则需要不断进行初始分组的调整，使得分组最后不再变化，类似K-means） + * @param key 用于分组的key + * @param similar 相似判断函数 + * @return 聚合后的分组结果 + * */ +fun Collection.groupBySimilar(key: (x1: E) -> String, similar: (x1: E, x2: E) -> Boolean): List> { + val sortedNodes = this.sortedBy { key(it) } + val groups = mutableListOf>() + var group = mutableListOf() + groups.add(group) + group.add(sortedNodes.first()) + var previousGroup: MutableList? = null + for (i in 1 until sortedNodes.size) { + val str2 = sortedNodes[i] + /* + * 一个新数据，只存在以下三种情况 + * 1. 加入当前组 + * 2. 中间存在干扰数据，加入之前的组 + * 3. 新建一个组 + * */ + if (similar(group.last(), str2)) { + group.add(str2) + } else if (previousGroup != null && similar(previousGroup.last(), str2)) { + previousGroup.add(str2) + } else { + if (group.size > 1) { + previousGroup = group + } + group = mutableListOf() + groups.add(group) + group.add(str2) + } + } + return groups +} diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/StringPool.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/StringPool.kt index 1f5e1edbde..a011534398 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/StringPool.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/StringPool.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.common.api.constant import java.net.URLDecoder import java.net.URLEncoder import java.nio.charset.StandardCharsets +import java.text.DecimalFormat import java.util.UUID import java.util.concurrent.ThreadLocalRandom import kotlin.math.abs @@ -73,14 +74,20 @@ object StringPool { suffix?.let { value += suffix } return value } + + fun calculateRatio(originSize: Long, newSize: Long, format: DecimalFormat = DecimalFormat("#.#")): String { + return format.format((originSize - newSize.toDouble()) / originSize * 100).plus("%") + } } fun String.ensurePrefix(prefix: CharSequence): String { return if (startsWith(prefix)) this else StringBuilder(prefix).append(this).toString() } + fun String.ensureSuffix(suffix: CharSequence): String { return if (endsWith(suffix)) this else this + suffix } + fun String.ensurePrefix(prefix: Char) = if (startsWith(prefix)) this else prefix + this fun String.ensureSuffix(suffix: Char) = if (endsWith(suffix)) this else this + suffix fun String.urlEncode() = URLEncoder.encode(this, StandardCharsets.UTF_8.displayName()).apply { diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt index 01062877b0..c6fbd7f183 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt @@ -33,7 +33,9 @@ package com.tencent.bkrepo.common.api.message enum class CommonMessageCode(private val key: String) : MessageCode { - SUCCESS("success") { override fun getCode() = 0 }, + SUCCESS("success") { + override fun getCode() = 0 + }, SYSTEM_ERROR("system.error"), PARAMETER_MISSING("system.parameter.missing"), @@ -44,6 +46,7 @@ enum class CommonMessageCode(private val key: String) : MessageCode { RESOURCE_NOT_FOUND("system.resource.not-found"), RESOURCE_EXPIRED("system.resource.expired"), RESOURCE_ARCHIVED("system.resource.archived"), + RESOURCE_COMPRESSED("system.resource.compressed"), METHOD_NOT_ALLOWED("system.method.not-allowed"), REQUEST_DENIED("system.request.denied"), REQUEST_UNAUTHENTICATED("system.request.unauthenticated"), @@ -56,7 +59,8 @@ enum class CommonMessageCode(private val key: String) : MessageCode { MODIFY_PASSWORD_FAILED("modify.password.failed"), OPERATION_CROSS_CLUSTER_NOT_ALLOWED("operation.cross-cluster.not-allowed"), MEDIA_TYPE_UNACCEPTABLE("system.media-type.unacceptable"), - TOO_MANY_REQUESTS("too.many.requests") + TOO_MANY_REQUESTS("too.many.requests"), + ; override fun getBusinessCode() = ordinal + 1 diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/stream/StreamExtensions.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/stream/StreamExtensions.kt new file mode 100644 index 0000000000..8d21f57c31 --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/stream/StreamExtensions.kt @@ -0,0 +1,22 @@ +package com.tencent.bkrepo.common.api.stream + +import com.google.common.primitives.Ints +import com.google.common.primitives.Longs +import java.io.IOException +import java.io.InputStream + +fun InputStream.readInt(): Int { + val bytes = ByteArray(4) + if (this.read(bytes) < 4) { + throw IOException("Not enough bytes to read for int.") + } + return Ints.fromByteArray(bytes) +} + +fun InputStream.readLong(): Long { + val bytes = ByteArray(8) + if (this.read(bytes) < 8) { + throw IOException("Not enough bytes to read for long.") + } + return Longs.fromByteArray(bytes) +} diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/StreamUtils.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/StreamUtils.kt index 5c3701b522..f018ae0872 100644 --- a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/StreamUtils.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/StreamUtils.kt @@ -1,6 +1,7 @@ package com.tencent.bkrepo.common.api.util import java.io.InputStream +import java.io.OutputStream import java.nio.charset.Charset /** @@ -29,4 +30,35 @@ object StreamUtils { fun InputStream.readText(charset: Charset = Charsets.UTF_8) = this.use { it.reader(charset).use { reader -> reader.readText() } } + + fun use( + inputStream: InputStream, + outputStream: OutputStream, + block: (input: InputStream, output: OutputStream) -> Unit, + ) { + inputStream.use { + outputStream.use { + block(inputStream, outputStream) + } + } + } + + fun useCopy( + inputStream: InputStream, + outputStream: OutputStream, + ) { + use(inputStream, outputStream) { input, output -> + input.copyTo(output) + } + } + + fun InputStream.drain(): Long { + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + var bytesRead: Int + var byteCount: Long = 0 + while (this.read(buffer).also { bytesRead = it } != -1) { + byteCount += bytesRead + } + return byteCount + } } diff --git a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/stream/Range.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/stream/Range.kt index 505573f068..2ea0a5cd5d 100644 --- a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/stream/Range.kt +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/stream/Range.kt @@ -39,25 +39,29 @@ import kotlin.math.min * @param endPosition 结束位置，最大值为[total]-1 * @param total 文件总长度 */ -class Range(startPosition: Long, endPosition: Long, val total: Long) { +class Range(startPosition: Long, endPosition: Long, val total: Long?) { /** * 起始位置 */ - val start: Long = if (startPosition < 0) 0 else startPosition + val start: Long = if (total == null || startPosition < 0) 0 else startPosition /** * 结束位置，范围为[start, total-1]，如果超出返回则设置为[total] - 1 */ - val end: Long = if (endPosition < 0) total - 1 else min(endPosition, total - 1) + val end: Long = if (total != null) { + if (endPosition < 0) total - 1 else min(endPosition, total - 1) + } else { + 0 + } /** * 范围长度 */ - val length: Long = end - start + 1 + val length: Long = if (total != null) end - start + 1 else 0 init { - require(total >= 0) { "Invalid total size: $total" } + require(total == null || total >= 0) { "Invalid total size: $total" } require(length >= 0) { "Invalid range length $length" } } @@ -65,14 +69,14 @@ class Range(startPosition: Long, endPosition: Long, val total: Long) { * 是否为部分内容 */ fun isPartialContent(): Boolean { - return length != total + return total != null && length != total } /** * 是否为完整内容 */ fun isFullContent(): Boolean { - return length == total + return total == null || length == total } /** @@ -91,5 +95,7 @@ class Range(startPosition: Long, endPosition: Long, val total: Long) { * 创建长度为[total]的完整范围 */ fun full(total: Long) = Range(0, total - 1, total) + + val FULL_RANGE = Range(0, 0, null) } } diff --git a/src/backend/common/common-artifact/artifact-api/src/test/kotlin/com/tencent/bkrepo/common/artifact/stream/RangeTest.kt b/src/backend/common/common-artifact/artifact-api/src/test/kotlin/com/tencent/bkrepo/common/artifact/stream/RangeTest.kt index 3e00e5a30e..77894cce64 100644 --- a/src/backend/common/common-artifact/artifact-api/src/test/kotlin/com/tencent/bkrepo/common/artifact/stream/RangeTest.kt +++ b/src/backend/common/common-artifact/artifact-api/src/test/kotlin/com/tencent/bkrepo/common/artifact/stream/RangeTest.kt @@ -55,5 +55,9 @@ internal class RangeTest { range = Range(0, 2505730872, 2505730873) Assertions.assertFalse(range.isPartialContent()) + + range = Range.FULL_RANGE + Assertions.assertTrue(range.isFullContent()) + Assertions.assertFalse(range.isPartialContent()) } } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt index 0bf3afc6c9..9b911b7596 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.common.artifact.manager +import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.artifact.manager.resource.FsNodeResource @@ -51,6 +52,7 @@ class NodeResourceFactoryImpl( private val storageCredentialsClient: StorageCredentialsClient, private val fsNodeClient: FsNodeClient, private val clusterNodeClient: ClusterNodeClient, + private val archiveClient: ArchiveClient, ) : NodeResourceFactory { private val centerClusterInfo = ClusterInfo( @@ -79,7 +81,14 @@ class NodeResourceFactoryImpl( ?: throw ErrorCodeException(ReplicationMessageCode.CLUSTER_NODE_NOT_FOUND, clusterName) return RemoteNodeResource(digest, range, storageCredentials, clusterInfo, storageService, false) } - return LocalNodeResource(nodeInfo, range, storageCredentials, storageService, storageCredentialsClient) + return LocalNodeResource( + nodeInfo, + range, + storageCredentials, + storageService, + storageCredentialsClient, + archiveClient, + ) } private fun isFsFile(node: NodeInfo): Boolean { diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt index 8d5ee66a42..a144eb5cec 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt @@ -27,8 +27,6 @@ package com.tencent.bkrepo.common.artifact.manager -import com.tencent.bkrepo.archive.api.ArchiveClient -import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode @@ -70,7 +68,6 @@ class StorageManager( private val nodeClient: NodeClient, private val nodeResourceFactoryImpl: NodeResourceFactoryImpl, private val pluginManager: PluginManager, - private val archiveClient: ArchiveClient, ) { /** @@ -127,19 +124,6 @@ class StorageManager( if (range.isEmpty() || request?.method == HttpMethod.HEAD.name) { return ArtifactInputStream(EmptyInputStream.INSTANCE, range) } - if (node.archived == true) { - try { - val restoreCreateArchiveFileRequest = ArchiveFileRequest( - sha256 = node.sha256!!, - storageCredentialsKey = storageCredentials?.key, - operator = SecurityUtils.getUserId(), - ) - archiveClient.restore(restoreCreateArchiveFileRequest) - } catch (e: Exception) { - logger.error("restore error", e) - } - throw ErrorCodeException(CommonMessageCode.RESOURCE_ARCHIVED, node.fullPath) - } val nodeResource = nodeResourceFactoryImpl.getNodeResource(node, range, storageCredentials) return nodeResource.getArtifactInputStream() } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/LocalNodeResource.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/LocalNodeResource.kt index 639cc9d084..fed0cdd898 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/LocalNodeResource.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/LocalNodeResource.kt @@ -27,9 +27,15 @@ package com.tencent.bkrepo.common.artifact.manager.resource +import com.tencent.bkrepo.archive.api.ArchiveClient +import com.tencent.bkrepo.archive.request.ArchiveFileRequest +import com.tencent.bkrepo.archive.request.UncompressFileRequest +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.repository.api.StorageCredentialsClient @@ -45,7 +51,8 @@ class LocalNodeResource( private val range: Range, private val storageCredentials: StorageCredentials?, private val storageService: StorageService, - private val storageCredentialsClient: StorageCredentialsClient + private val storageCredentialsClient: StorageCredentialsClient, + private val archiveClient: ArchiveClient, ) : AbstractNodeResource() { private val digest = node.sha256.orEmpty() @@ -63,6 +70,21 @@ class LocalNodeResource( return storageService.load(digest, range, storageCredentials) ?: loadFromCopyIfNecessary(node, range) ?: loadFromRepoOldIfNecessary(node, range, storageCredentials) + ?: let { + /* + * 为了避免在存储时，处理新上传的文件与已归档或者已压缩的文件相同时的情况（存多，归档/压缩少）， + * 我们选择在load的时候进行归档或者压缩文件的处理，因为读取不到文件的情况较少，所以这样产生的额外消耗更少 + * */ + if (node.archived == true) { + restore(node, storageCredentials) + throw ErrorCodeException(CommonMessageCode.RESOURCE_ARCHIVED, node.fullPath) + } + if (node.compressed == true) { + uncompress(node, storageCredentials) + throw ErrorCodeException(CommonMessageCode.RESOURCE_COMPRESSED, node.fullPath) + } + null + } } /** @@ -71,7 +93,7 @@ class LocalNodeResource( * */ private fun loadFromCopyIfNecessary( node: NodeInfo, - range: Range + range: Range, ): ArtifactInputStream? { node.copyFromCredentialsKey?.let { val digest = node.sha256!! @@ -89,14 +111,14 @@ class LocalNodeResource( private fun loadFromRepoOldIfNecessary( node: NodeInfo, range: Range, - storageCredentials: StorageCredentials? + storageCredentials: StorageCredentials?, ): ArtifactInputStream? { val repositoryDetail = getRepoDetail(node) val oldCredentials = findStorageCredentialsByKey(repositoryDetail.oldCredentialsKey) if (storageCredentials != oldCredentials) { logger.info( "load data [${node.sha256!!}] from" + - " repo old credentialsKey [${repositoryDetail.oldCredentialsKey}]" + " repo old credentialsKey [${repositoryDetail.oldCredentialsKey}]", ) return storageService.load(node.sha256!!, range, oldCredentials) } @@ -117,7 +139,7 @@ class LocalNodeResource( // 如果是异步或者请求上下文找不到，则通过查询，并进行缓存 val repositoryId = ArtifactContextHolder.RepositoryId( projectId = projectId, - repoName = repoName + repoName = repoName, ) return ArtifactContextHolder.getRepoDetail(repositoryId) } @@ -131,6 +153,32 @@ class LocalNodeResource( return storageCredentialsClient.findByKey(credentialsKey).data } + private fun restore(node: NodeInfo, storageCredentials: StorageCredentials?) { + try { + val restoreCreateArchiveFileRequest = ArchiveFileRequest( + sha256 = node.sha256!!, + storageCredentialsKey = storageCredentials?.key, + operator = SecurityUtils.getUserId(), + ) + archiveClient.restore(restoreCreateArchiveFileRequest) + } catch (e: Exception) { + logger.error("Restore error", e) + } + } + + private fun uncompress(node: NodeInfo, storageCredentials: StorageCredentials?) { + try { + val uncompressFileRequest = UncompressFileRequest( + node.sha256!!, + storageCredentialsKey = storageCredentials?.key, + operator = SecurityUtils.getUserId(), + ) + archiveClient.uncompress(uncompressFileRequest) + } catch (e: Exception) { + logger.error("Uncompress error", e) + } + } + companion object { private val logger = LoggerFactory.getLogger(LocalNodeResource::class.java) } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/CosRedirectService.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/CosRedirectService.kt index ef5f8009dd..b20135b677 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/CosRedirectService.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/redirect/CosRedirectService.kt @@ -79,7 +79,12 @@ class CosRedirectService( // 从request uri中获取artifact信息，artifact为null时表示非单制品下载请求，此时不支持重定向 val artifact = ArtifactContextHolder.getArtifactInfo() // node为null时表示制品不存在，或者是Remote仓库的制品尚未被缓存，此时不支持重定向 - if (node == null || node.folder || artifact == null) { + if (node == null || + node.folder || + artifact == null || + node.compressed == true || // 压缩文件不支持重定向 + node.archived == true // 归档文件不支持重定向 + ) { return false } @@ -102,8 +107,8 @@ class CosRedirectService( val redirectTo = HttpContextHolder.getRequest().getHeader("X-BKREPO-DOWNLOAD-REDIRECT-TO") val needToRedirect = repoSupportRedirectTo || - redirectTo == RedirectTo.INNERCOS.name || - storageProperties.redirect.redirectAllDownload + redirectTo == RedirectTo.INNERCOS.name || + storageProperties.redirect.redirectAllDownload // 文件存在于COS上时才会被重定向 return needToRedirect && isSystemOrAdmin() && guessFileExists(node, storageCredentials) @@ -148,7 +153,7 @@ class CosRedirectService( logger.warn("Failed to resolve http range: ${exception.message}") throw ErrorCodeException( status = HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE, - messageCode = CommonMessageCode.REQUEST_RANGE_INVALID + messageCode = CommonMessageCode.REQUEST_RANGE_INVALID, ) } } diff --git a/src/backend/common/common-bksync/build.gradle.kts b/src/backend/common/common-bksync/build.gradle.kts index 5ee1f0d422..5986adbfab 100644 --- a/src/backend/common/common-bksync/build.gradle.kts +++ b/src/backend/common/common-bksync/build.gradle.kts @@ -4,6 +4,7 @@ dependencies { implementation(project(":common:common-api")) implementation("ch.qos.logback:logback-classic") implementation("com.squareup.okhttp3:okhttp-sse:${Versions.OKhttp}") + implementation("commons-codec:commons-codec") } plugins { id("me.champeau.gradle.jmh") version Versions.JMH diff --git a/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/BkSync.kt b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/BkSync.kt index 977f79a468..825d1e0462 100644 --- a/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/BkSync.kt +++ b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/BkSync.kt @@ -45,7 +45,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int * @param checksumOutput 校验和输出流 * */ fun checksum(file: File, checksumOutput: OutputStream) { - checksum(file.inputStream(), checksumOutput) + file.inputStream().use { checksum(it, checksumOutput) } } /** @@ -91,10 +91,14 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int fun diff(file: File, checksumStream: InputStream, deltaOutput: OutputStream, reuseThreshold: Float): DiffResult { // 使用滑动窗口检测,找到与远端相同的部分 val index = ChecksumIndex(checksumStream) - val window = BufferedSlidingWindow(blockSize, windowBufferSize, file.inputStream(), file.length()) val raf = RandomAccessFile(file, READ) - raf.use { - return detecting(window, index, deltaOutput, it, reuseThreshold) + val srcInput = file.inputStream() + try { + val window = BufferedSlidingWindow(blockSize, windowBufferSize, srcInput, file.length()) + return detecting(window, index, deltaOutput, raf, reuseThreshold) + } finally { + raf.close() + srcInput.close() } } @@ -111,7 +115,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int index: ChecksumIndex, outputStream: OutputStream, raf: RandomAccessFile, - reuseThreshold: Float + reuseThreshold: Float, ): DiffResult { var content: ByteArray var deltaStart: Long @@ -166,7 +170,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int private fun search( rollingHash: Int, index: ChecksumIndex, - slidingWindow: BufferedSlidingWindow + slidingWindow: BufferedSlidingWindow, ): Checksum? { if (!index.exist(rollingHash)) { return null @@ -182,7 +186,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int deltaStart: Long, deltaEnd: Long, raf: RandomAccessFile, - outputStream: OutputStream + outputStream: OutputStream, ) { val len = deltaEnd - deltaStart if (len > Int.MAX_VALUE) { @@ -207,7 +211,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int adler32RollingHash: Adler32RollingHash, index: ChecksumIndex, reuse: Int, - reuseThreshold: Float + reuseThreshold: Float, ): Checksum? { // 达到重复使用阈值需要的数据字节数 val detectingDataCount = ceil(index.total * reuseThreshold - reuse) * slidingWindow.windowSize.toLong() @@ -260,7 +264,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int outputStream: OutputStream, len: Long, ras: RandomAccessFile, - deltaStart: Long + deltaStart: Long, ) { // 写入数据流标志-1 outputStream.write(BEGIN_FLAG) @@ -300,7 +304,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int fun merge( blockChannel: BlockChannel, deltaInput: InputStream, - newFileChannel: WritableByteChannel + newFileChannel: WritableByteChannel, ): MergeResult { val deltaStream = DeltaInputStream(deltaInput) var reuse = 0 @@ -360,7 +364,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int blockChannel.name(), blockSize, sha256, - md5 + md5, ) logger.info(mergeResult.toString()) return mergeResult @@ -377,7 +381,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int newFileChannel: WritableByteChannel, startSeq: Int, endSeq: Int, - blockChannel: BlockChannel + blockChannel: BlockChannel, ) { var currentStartSeq = startSeq while (currentStartSeq <= endSeq) { @@ -403,7 +407,7 @@ class BkSync(val blockSize: Int = DEFAULT_BLOCK_SIZE, var windowBufferSize: Int private fun copyDataSequence( newFileChannel: WritableByteChannel, deltaStream: DeltaInputStream, - len: Int + len: Int, ) { val bufferSize = len.coerceAtMost(DEFAULT_BUFFER_SIZE) var buffer = ByteArray(bufferSize) diff --git a/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BDUtils.kt b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BDUtils.kt new file mode 100644 index 0000000000..7cc2c28ecb --- /dev/null +++ b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BDUtils.kt @@ -0,0 +1,155 @@ +package com.tencent.bkrepo.common.bksync.file + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.util.StreamUtils +import com.tencent.bkrepo.common.api.util.StreamUtils.drain +import com.tencent.bkrepo.common.bksync.BkSync +import com.tencent.bkrepo.common.bksync.file.BkSyncDeltaSource.Companion.toBkSyncDeltaSource +import com.tencent.bkrepo.common.bksync.transfer.exception.InterruptedRollingException +import com.tencent.bkrepo.common.bksync.transfer.exception.TooLowerReuseRateException +import java.io.File +import java.nio.channels.FileChannel +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import java.security.DigestInputStream +import java.security.MessageDigest +import org.slf4j.LoggerFactory + +/** + * BD压缩工具 + * + * 使用bksync实现的增量压缩和解压功能 + * */ +object BDUtils { + + private const val BK_SYNC_FILE_PREFIX = "bksync_" + private const val BK_SYNC_FILE_SUFFIX = ".temp" + private val logger = LoggerFactory.getLogger(BDUtils::class.java) + + /** + * 增量压缩 + * + * 基于bksync实现增量压缩功能，压缩后生成bd格式文件 + * @param src 源文件 + * @param dest 目标文件 + * @param srcKey 源文件的key + * @param destKey 目标文件的key + * @param workDir 工作目录 + * @param threshold 源文件和目标文件最低重复率阈值 + * @return 压缩后的文件 + * @throws InterruptedRollingException 重复率低于[threshold]则抛出该异常 + * */ + fun delta( + src: File, + dest: File, + srcKey: String, + destKey: String, + workDir: Path, + threshold: Float, + ): File { + // 源md5 + val bkSync = BkSync() + val path = workDir.resolve(BK_SYNC_FILE_PREFIX) + try { + // 签名文件 + val signFile = path.createTempFile() + signFile.outputStream().use { bkSync.checksum(dest, it) } + return deltaByChecksumFile(src, signFile, srcKey, destKey, workDir, threshold) + } finally { + path.toFile().deleteRecursively() + } + } + + /** + * 增量压缩 + * + * @param src 源文件 + * @param checksumFile 签名文件 + * @param srcKey 源文件的key + * @param destKey 目标文件的key + * @param workDir 工作目录 + * @param threshold 源文件和目标文件最低重复率阈值 + * @return 压缩后的文件 + * @throws InterruptedRollingException 重复率低于[threshold]则抛出该异常 + * */ + fun deltaByChecksumFile( + src: File, + checksumFile: File, + srcKey: String, + destKey: String, + workDir: Path, + threshold: Float, + ): File { + val bkSync = BkSync() + val path = workDir.resolve(BK_SYNC_FILE_PREFIX) + try { + /* + * 压缩过程 + * 1. 根据src和dest生成增量文件 + * 2. 计算原文件md5，并构建bd source + * 3. 将bd source写入文件，生成bd文件 + * */ + val deltaFile = path.createTempFile() + StreamUtils.use(checksumFile.inputStream(), deltaFile.outputStream()) { input, output -> + val result = bkSync.diff(src, input, output, threshold) + if (result.hitRate < threshold) { + logger.info("Repeat rate[${result.hitRate}] is below threshold[$threshold],stop detection.") + throw TooLowerReuseRateException() + } + } + val messageDigest = MessageDigest.getInstance("MD5") + DigestInputStream(src.inputStream(), messageDigest).use { it.drain() } + val source = FileBkSyncDeltaSource(srcKey, destKey, messageDigest.digest(), deltaFile) + val file = workDir.createTempFile() + file.outputStream().use { source.writeTo(it) } + return file + } finally { + path.toFile().deleteRecursively() + } + } + + /** + * 解压文件 + * @param bdFile bd压缩文件 + * @param dest 目标文件 + * @param workDir 工作目录 + * @return 源文件 + * @throws IllegalStateException md5校验失败，则抛出该异常 + * */ + fun patch(bdFile: File, dest: File, workDir: Path): File { + val path = workDir.resolve(BK_SYNC_FILE_PREFIX) + val file = path.createTempFile() + try { + /* + * 解压过程 + * 1. 读取bd文件 + * 2. 通过bksync合并delta和dest,合并生成源文件 + * 3. 校验源文件md5 + * */ + val bdSource = bdFile.toBkSyncDeltaSource(file) + val srcFile = workDir.createTempFile() + FileChannel.open(srcFile.toPath(), StandardOpenOption.WRITE).use { srcChannel -> + bdSource.content().use { deltaInput -> + val bksync = BkSync() + bksync.calculateMd5 = true + val result = bksync.merge(dest, deltaInput, srcChannel) + check(result.md5 == bdSource.getSrcMd5()) { "File [${bdSource.src}] failed verification" } + } + } + return srcFile + } finally { + path.toFile().deleteRecursively() + } + } + + /** + * 创建临时文件 + * */ + private fun Path.createTempFile(): File { + val srcFileName = StringPool.randomStringByLongValue(BK_SYNC_FILE_PREFIX, BK_SYNC_FILE_SUFFIX) + val path = this.resolve(srcFileName) + Files.createDirectories(path.parent) + return Files.createFile(path).toFile() + } +} diff --git a/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaFileHeader.kt b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaFileHeader.kt new file mode 100644 index 0000000000..e52f3d61ea --- /dev/null +++ b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaFileHeader.kt @@ -0,0 +1,22 @@ +package com.tencent.bkrepo.common.bksync.file + +/** + * BD资源头 + * */ +class BkSyncDeltaFileHeader( + val src: String, + val dest: String, + val md5Bytes: ByteArray, + val dataSize: Long, + val extra: Byte, +) { + /** + * 文件长度 + * */ + val size = src.toByteArray().size + dest.toByteArray().size + dataSize + STATIC_LENGTH + + companion object { + // bd文件头静态部分固定长度 + const val STATIC_LENGTH = 41 + } +} diff --git a/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaSource.kt b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaSource.kt new file mode 100644 index 0000000000..d94ce07049 --- /dev/null +++ b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaSource.kt @@ -0,0 +1,189 @@ +package com.tencent.bkrepo.common.bksync.file + +import com.google.common.primitives.Ints +import com.google.common.primitives.Longs +import com.tencent.bkrepo.common.api.stream.readInt +import com.tencent.bkrepo.common.api.stream.readLong +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.InputStream +import java.io.OutputStream +import java.util.zip.CRC32 +import org.apache.commons.codec.binary.Hex + +/** + * bd抽象资源 + * + * 定义了bd资源的格式，支持输出和读取bd资源。 + * bd资源可用于增量压缩中，记录了源文件和目标文件信息，然后 + * 通过这些信息，可以还原源文件，同时支持md5. + * 另外bd资源也可以保存其他相关信息，只要需要记录两个key之间关系 + * 都可以用该格式 + * */ +abstract class BkSyncDeltaSource( + val src: String, // 源key + val dest: String, // 目标key + val md5Bytes: ByteArray, // 源md5 +) { + + /** + * 校验bd资源完整性 + * */ + private val crc32 = CRC32() + private val srcBytes = src.toByteArray() + private val destBytes = dest.toByteArray() + private val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + + /** + * bd资源内容 + * */ + abstract fun content(): InputStream + + /** + * bd资源内容长度 + * */ + abstract fun contentLength(): Long + + /** + * 将该资源写入到[outputStream] + * + * bd格式分为三部分 + * 1. header 保存bd文件的基本信息，比如src/dest + * 2. payload 实际内容 + * 3. trailer crc32 + * */ + fun writeTo(outputStream: OutputStream) { + content().use { + crc32.reset() + outputStream.write(encodeHeader()) + var bytes = it.read(buffer) + var bytesCopied = 0 + while (bytes >= 0) { + crc32.update(buffer, 0, bytes) + outputStream.write(buffer, 0, bytes) + bytesCopied += bytes + bytes = it.read(buffer) + } + outputStream.write(encodeTrailer()) + } + } + + /** + * 获取bd资源大小 + * */ + fun getSize(): Long { + return srcBytes.size + destBytes.size + contentLength() + BkSyncDeltaFileHeader.STATIC_LENGTH + } + + /** + * 获取源md5 + * */ + fun getSrcMd5(): String { + return Hex.encodeHexString(md5Bytes) + } + + /** + * 编码header + * + * header格式 + * Magic number -4B + * src and dest sum size -4B (src size in high 16bit,dest size in low 16bit) + * src -src size B + * dest -dest size B + * src md5 -16B + * data size -8B + * extra -1B + * */ + private fun encodeHeader(): ByteArray { + val headerOutput = ByteArrayOutputStream() + headerOutput.write(Ints.toByteArray(BD_MAGIC)) + headerOutput.write(Ints.toByteArray(srcBytes.size.shl(16).or(destBytes.size))) + headerOutput.write(srcBytes) + headerOutput.write(destBytes) + headerOutput.write(md5Bytes) + headerOutput.write(Longs.toByteArray(contentLength())) + headerOutput.write(0) + return headerOutput.toByteArray() + } + + /** + * 编码尾部 + * */ + private fun encodeTrailer(): ByteArray { + return Longs.toByteArray(crc32.value) + } + + companion object { + // BD资源魔术，2023bd + const val BD_MAGIC = 0x14176264 + + /** + * 读取BD文件 + * @param contentFile BD内容保存文件 + * @return BD资源 + * */ + fun File.toBkSyncDeltaSource(contentFile: File): BkSyncDeltaSource { + return this.inputStream().use { it.toBkSyncDeltaSource(contentFile) } + } + + /** + * 读取BD数据流 + * @param contentFile BD内容保存文件 + * @return BD资源 + * */ + fun InputStream.toBkSyncDeltaSource(contentFile: File): BkSyncDeltaSource { + val header = readHeader(this) + var remaining = header.dataSize + val crc32 = CRC32() + var buffer = ByteArray(remaining.toInt().coerceAtMost(DEFAULT_BUFFER_SIZE)) + var bytes = this.read(buffer) + contentFile.outputStream().use { + while (bytes >= 0 && remaining > 0) { + crc32.update(buffer, 0, bytes) + it.write(buffer, 0, bytes) + remaining -= bytes + if (remaining < buffer.size) { + buffer = ByteArray(remaining.toInt()) + } + bytes = this.read(buffer) + } + } + readTrailer(this, crc32) + with(header) { + return FileBkSyncDeltaSource(src, dest, md5Bytes, contentFile) + } + } + + /** + * 读取BD头 + * @param inputStream BD流 + * @return BD头 + * */ + fun readHeader(inputStream: InputStream): BkSyncDeltaFileHeader { + val magicNum = inputStream.readInt() + require(magicNum == BD_MAGIC) { "Not in BD format" } + val sumLen = inputStream.readInt() + val srcSize = sumLen.ushr(16) + val destSize = 0xFFFF and sumLen + require(srcSize > 0 && destSize > 0) + // 剩余header大小 + val headerPayloadSize = srcSize + destSize + 25 + val bytes = ByteArray(headerPayloadSize) + require(inputStream.read(bytes) == headerPayloadSize) { "Corrupt BD header" } + val src = String(bytes, 0, srcSize) + val dest = String(bytes, srcSize, destSize) + val md5Bytes = bytes.copyOfRange(bytes.lastIndex - 24, bytes.lastIndex - 8) + val dataSize = Longs.fromByteArray(bytes.copyOfRange(bytes.lastIndex - 8, bytes.lastIndex)) + val extra = bytes.last() + require(dataSize > 0) + return BkSyncDeltaFileHeader(src, dest, md5Bytes, dataSize, extra) + } + + /** + * 读取尾部 + * */ + private fun readTrailer(inputStream: InputStream, crc32: CRC32) { + require(inputStream.readLong() == crc32.value) { "Corrupt BD trailer" } + } + } +} diff --git a/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/ByteArrayBkSyncDeltaSource.kt b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/ByteArrayBkSyncDeltaSource.kt new file mode 100644 index 0000000000..5491663c70 --- /dev/null +++ b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/ByteArrayBkSyncDeltaSource.kt @@ -0,0 +1,21 @@ +package com.tencent.bkrepo.common.bksync.file + +import java.io.InputStream + +/** + * 字节数组实现的BD资源 + * */ +class ByteArrayBkSyncDeltaSource( + src: String, + dest: String, + srcMd5: ByteArray, + val bytes: ByteArray, +) : BkSyncDeltaSource(src, dest, srcMd5) { + override fun content(): InputStream { + return bytes.inputStream() + } + + override fun contentLength(): Long { + return bytes.size.toLong() + } +} diff --git a/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/FileBkSyncDeltaSource.kt b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/FileBkSyncDeltaSource.kt new file mode 100644 index 0000000000..ddf9cef388 --- /dev/null +++ b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/file/FileBkSyncDeltaSource.kt @@ -0,0 +1,22 @@ +package com.tencent.bkrepo.common.bksync.file + +import java.io.File +import java.io.InputStream + +/** + * BD文件 + * */ +class FileBkSyncDeltaSource( + src: String, + dest: String, + srcMd5: ByteArray, + val file: File, +) : BkSyncDeltaSource(src, dest, srcMd5) { + override fun content(): InputStream { + return file.inputStream() + } + + override fun contentLength(): Long { + return file.length() + } +} diff --git a/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/transfer/exception/TooLowerReuseRateException.kt b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/transfer/exception/TooLowerReuseRateException.kt new file mode 100644 index 0000000000..b102a5a06a --- /dev/null +++ b/src/backend/common/common-bksync/src/main/kotlin/com/tencent/bkrepo/common/bksync/transfer/exception/TooLowerReuseRateException.kt @@ -0,0 +1,6 @@ +package com.tencent.bkrepo.common.bksync.transfer.exception + +/** + * 重复率过低异常 + * */ +class TooLowerReuseRateException : RuntimeException() diff --git a/src/backend/common/common-bksync/src/test/kotlin/com/tencent/bkrepo/common/bksync/file/BDUtilsTest.kt b/src/backend/common/common-bksync/src/test/kotlin/com/tencent/bkrepo/common/bksync/file/BDUtilsTest.kt new file mode 100644 index 0000000000..6743a74e0f --- /dev/null +++ b/src/backend/common/common-bksync/src/test/kotlin/com/tencent/bkrepo/common/bksync/file/BDUtilsTest.kt @@ -0,0 +1,54 @@ +package com.tencent.bkrepo.common.bksync.file + +import com.tencent.bkrepo.common.bksync.file.BkSyncDeltaSource.Companion.toBkSyncDeltaSource +import java.nio.file.Files +import java.nio.file.Paths +import java.security.MessageDigest +import kotlin.random.Random +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class BDUtilsTest { + private val worDir = Paths.get(System.getProperty("java.io.tmpdir"), "bksync-ut") + + @BeforeEach + fun init() { + Files.createDirectories(worDir) + } + + @AfterEach + fun cleanup() { + worDir.toFile().deleteRecursively() + } + + @Test + fun deltaAndPatchTest() { + val data1 = Random.nextBytes(Random.nextInt(1024, 1 shl 20)) + val data2 = data1.copyOfRange(Random.nextInt(1, 10), data1.size) + val tempFile1 = createTempFile() + tempFile1.writeBytes(data1) + val tempFile2 = createTempFile() + tempFile2.writeBytes(data2) + val bdFile = BDUtils.delta(tempFile1, tempFile2, "srcKey", "destKey", worDir, 0.8f) + val deltaFile = createTempFile() + val bdSource = bdFile.toBkSyncDeltaSource(deltaFile) + Assertions.assertEquals("srcKey", bdSource.src) + Assertions.assertEquals("destKey", bdSource.dest) + Assertions.assertArrayEquals(MessageDigest.getInstance("MD5").digest(data1), bdSource.md5Bytes) + + // 正常恢复 + val srcFile = BDUtils.patch(bdFile, tempFile2, worDir) + Assertions.assertArrayEquals(data1, srcFile.readBytes()) + + // 恢复出错 + Assertions.assertThrows(IllegalStateException::class.java) { + BDUtils.patch( + bdFile, + tempFile1, + worDir, + ) + } + } +} diff --git a/src/backend/common/common-bksync/src/test/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaSourceTest.kt b/src/backend/common/common-bksync/src/test/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaSourceTest.kt new file mode 100644 index 0000000000..f6591994a3 --- /dev/null +++ b/src/backend/common/common-bksync/src/test/kotlin/com/tencent/bkrepo/common/bksync/file/BkSyncDeltaSourceTest.kt @@ -0,0 +1,97 @@ +package com.tencent.bkrepo.common.bksync.file + +import com.tencent.bkrepo.common.api.stream.readInt +import com.tencent.bkrepo.common.api.stream.readLong +import com.tencent.bkrepo.common.bksync.file.BkSyncDeltaSource.Companion.toBkSyncDeltaSource +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.util.zip.CRC32 +import kotlin.random.Random +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class BkSyncDeltaSourceTest { + @Test + fun writeTest() { + val data = Random.nextBytes(Random.nextInt(8 shl 20)) + val srcMd5bytes = Random.nextBytes(16) + val bd = ByteArrayBkSyncDeltaSource( + src = "bd-test-src", + dest = "bk-test-dest", + srcMd5 = srcMd5bytes, + bytes = data, + ) + val outputStream = ByteArrayOutputStream() + bd.writeTo(outputStream) + val encodedData = outputStream.toByteArray() + // 校验大小 + Assertions.assertTrue(bd.getSize() == encodedData.size.toLong()) + // 校验header + val inputStream = ByteArrayInputStream(encodedData) + // magic number + Assertions.assertEquals(BkSyncDeltaSource.BD_MAGIC, inputStream.readInt()) + + // key + val nameLen = bd.src.length.shl(16) or bd.dest.length + Assertions.assertEquals(nameLen, inputStream.readInt()) + val srcLen = nameLen.ushr(16) + val destLen = 0xFFFF and nameLen + Assertions.assertEquals(bd.src.length, srcLen) + Assertions.assertEquals(bd.dest.length, destLen) + var bytes = ByteArray(srcLen + destLen) + val read = inputStream.read(bytes) + Assertions.assertEquals(bytes.size, read) + val src = String(bytes, 0, srcLen) + val dest = String(bytes, srcLen, destLen) + Assertions.assertEquals(bd.src, src) + Assertions.assertEquals(bd.dest, dest) + + // md5 + val md5Bytes = ByteArray(16) + inputStream.read(md5Bytes) + Assertions.assertArrayEquals(srcMd5bytes, md5Bytes) + + // 校验body + Assertions.assertEquals(data.size.toLong(), inputStream.readLong()) + // extra + Assertions.assertEquals(0, inputStream.read()) + bytes = ByteArray(data.size) + inputStream.read(bytes) + Assertions.assertArrayEquals(data, bytes) + + // 校验trailer + val crC32 = CRC32() + crC32.update(bytes) + Assertions.assertEquals(crC32.value, inputStream.readLong()) + } + + @Test + fun readTest() { + val data = Random.nextBytes(Random.nextInt(8 shl 20)) + val srcMd5bytes = Random.nextBytes(16) + val bd = ByteArrayBkSyncDeltaSource( + src = "bd-test-src", + dest = "bk-test-dest", + srcMd5 = srcMd5bytes, + bytes = data, + ) + val outputStream = ByteArrayOutputStream() + bd.writeTo(outputStream) + val inputStream = ByteArrayInputStream(outputStream.toByteArray()) + val file = createTempFile() + try { + val readBdSource = inputStream.toBkSyncDeltaSource(file) + Assertions.assertEquals(bd.src, readBdSource.src) + Assertions.assertEquals(bd.dest, readBdSource.dest) + Assertions.assertArrayEquals(bd.md5Bytes, readBdSource.md5Bytes) + Assertions.assertEquals(bd.getSize(), readBdSource.getSize()) + Assertions.assertEquals(bd.contentLength(), readBdSource.contentLength()) + Assertions.assertEquals(bd.contentLength(), file.length()) + val data1 = ByteArray(data.size) + readBdSource.content().use { it.read(data1) } + Assertions.assertArrayEquals(data, data1) + } finally { + file.delete() + } + } +} diff --git a/src/backend/common/common-service/src/main/resources/i18n/messages_en.properties b/src/backend/common/common-service/src/main/resources/i18n/messages_en.properties index 33541388c3..3dc42dfe3a 100644 --- a/src/backend/common/common-service/src/main/resources/i18n/messages_en.properties +++ b/src/backend/common/common-service/src/main/resources/i18n/messages_en.properties @@ -36,6 +36,7 @@ system.parameter.invalid=Parameter [{0}] is invalid system.request.content.invalid=Invalid request content system.resource.existed=Resource [{0}] existed system.resource.archived=Resource [{0}] archived,Restore it will take about 12-24 hours, please wait patiently. +system.resource.compressed=Resource [{0}] compressed,uncompressing it, please wait patiently. system.resource.not-found=Resource [{0}] not found system.resource.expired=Resource [{0}] expired system.method.not-allowed=Method not allowed, reason: {0} diff --git a/src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties index 63533ddd76..7b377e738b 100644 --- a/src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/common/common-service/src/main/resources/i18n/messages_zh_CN.properties @@ -36,6 +36,7 @@ system.parameter.invalid=参数[{0}]无效 system.request.content.invalid=请求内容无效 system.resource.exist=资源[{0}]已存在 system.resource.archived=资源[{0}]已归档,恢复它大概需要12-24小时，请耐心等候 +system.resource.compressed=资源[{0}]已压缩,正在解压，请耐心等候 system.resource.not-found=资源[{0}]不存在 system.resource.expired=资源[{0}]已过期 system.method.not-allowed=不支持的操作, 原因: {0} diff --git a/src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties index 7b3ec2f262..8f64e2844a 100644 --- a/src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/common/common-service/src/main/resources/i18n/messages_zh_TW.properties @@ -38,6 +38,7 @@ system.resource.existed=資源[{0}]已存在 system.resource.not-found=資源[{0}]不存在 system.resource.expired=資源[{0}]已過期 system.resource.archived=資源[{0}]已歸檔，恢復它大概需要12-24小時，請耐心等候 +system.resource.compressed=資源[{0}]已歸檔，正在解壓縮，請耐心等候 system.method.not-allowed=不支持的操作, 原因: {0} system.request.denied=訪問被拒絕: {0} system.request.unauthenticated=認證失敗: {0} diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/CompressProperties.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/CompressProperties.kt new file mode 100644 index 0000000000..b5357eabb8 --- /dev/null +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/config/CompressProperties.kt @@ -0,0 +1,8 @@ +package com.tencent.bkrepo.common.storage.config + +data class CompressProperties( + /** + * 重复率，只有超过该值，文件才会被压缩 + * */ + val ratio: Float = 0.5f, +) diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/FileSystemCredentials.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/FileSystemCredentials.kt index a2c255414b..1e69f3d6ca 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/FileSystemCredentials.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/FileSystemCredentials.kt @@ -32,6 +32,7 @@ package com.tencent.bkrepo.common.storage.credentials import com.tencent.bkrepo.common.storage.config.CacheProperties +import com.tencent.bkrepo.common.storage.config.CompressProperties import com.tencent.bkrepo.common.storage.config.EncryptProperties import com.tencent.bkrepo.common.storage.config.UploadProperties @@ -43,8 +44,9 @@ data class FileSystemCredentials( override var key: String? = null, override var cache: CacheProperties = CacheProperties(), override var upload: UploadProperties = UploadProperties(), - override var encrypt: EncryptProperties = EncryptProperties() -) : StorageCredentials(key, cache, upload, encrypt) { + override var encrypt: EncryptProperties = EncryptProperties(), + override var compress: CompressProperties = CompressProperties(), +) : StorageCredentials(key, cache, upload, encrypt, compress) { companion object { const val type = "filesystem" diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/HDFSCredentials.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/HDFSCredentials.kt index df8f91c5ce..778b9508c0 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/HDFSCredentials.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/HDFSCredentials.kt @@ -32,6 +32,7 @@ package com.tencent.bkrepo.common.storage.credentials import com.tencent.bkrepo.common.storage.config.CacheProperties +import com.tencent.bkrepo.common.storage.config.CompressProperties import com.tencent.bkrepo.common.storage.config.EncryptProperties import com.tencent.bkrepo.common.storage.config.UploadProperties @@ -48,8 +49,9 @@ data class HDFSCredentials( override var key: String? = null, override var cache: CacheProperties = CacheProperties(), override var upload: UploadProperties = UploadProperties(), - override var encrypt: EncryptProperties = EncryptProperties() -) : StorageCredentials(key, cache, upload, encrypt) { + override var encrypt: EncryptProperties = EncryptProperties(), + override var compress: CompressProperties = CompressProperties(), +) : StorageCredentials(key, cache, upload, encrypt, compress) { companion object { const val type = "hdfs" } diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/InnerCosCredentials.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/InnerCosCredentials.kt index 6393e651c6..8922167d98 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/InnerCosCredentials.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/InnerCosCredentials.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.common.storage.credentials import com.tencent.bkrepo.common.operate.api.annotation.Sensitive import com.tencent.bkrepo.common.operate.api.handler.MaskPartString import com.tencent.bkrepo.common.storage.config.CacheProperties +import com.tencent.bkrepo.common.storage.config.CompressProperties import com.tencent.bkrepo.common.storage.config.EncryptProperties import com.tencent.bkrepo.common.storage.config.UploadProperties @@ -57,8 +58,9 @@ data class InnerCosCredentials( override var key: String? = null, override var cache: CacheProperties = CacheProperties(), override var upload: UploadProperties = UploadProperties(), - override var encrypt: EncryptProperties = EncryptProperties() -) : StorageCredentials(key, cache, upload, encrypt) { + override var encrypt: EncryptProperties = EncryptProperties(), + override var compress: CompressProperties = CompressProperties(), +) : StorageCredentials(key, cache, upload, encrypt, compress) { companion object { const val type = "innercos" diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/S3Credentials.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/S3Credentials.kt index 72641ecd22..8f860f22d3 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/S3Credentials.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/S3Credentials.kt @@ -32,6 +32,7 @@ package com.tencent.bkrepo.common.storage.credentials import com.tencent.bkrepo.common.storage.config.CacheProperties +import com.tencent.bkrepo.common.storage.config.CompressProperties import com.tencent.bkrepo.common.storage.config.EncryptProperties import com.tencent.bkrepo.common.storage.config.UploadProperties @@ -47,8 +48,9 @@ data class S3Credentials( override var key: String? = null, override var cache: CacheProperties = CacheProperties(), override var upload: UploadProperties = UploadProperties(), - override var encrypt: EncryptProperties = EncryptProperties() -) : StorageCredentials(key, cache, upload, encrypt) { + override var encrypt: EncryptProperties = EncryptProperties(), + override var compress: CompressProperties = CompressProperties(), +) : StorageCredentials(key, cache, upload, encrypt, compress) { companion object { const val type = "s3" } diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/StorageCredentials.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/StorageCredentials.kt index 70e2b88905..05b6ca1389 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/StorageCredentials.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/credentials/StorageCredentials.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.common.storage.credentials import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo import com.tencent.bkrepo.common.storage.config.CacheProperties +import com.tencent.bkrepo.common.storage.config.CompressProperties import com.tencent.bkrepo.common.storage.config.EncryptProperties import com.tencent.bkrepo.common.storage.config.UploadProperties @@ -44,11 +45,12 @@ import com.tencent.bkrepo.common.storage.config.UploadProperties @JsonSubTypes( JsonSubTypes.Type(value = FileSystemCredentials::class, name = FileSystemCredentials.type), JsonSubTypes.Type(value = InnerCosCredentials::class, name = InnerCosCredentials.type), - JsonSubTypes.Type(value = HDFSCredentials::class, name = HDFSCredentials.type) + JsonSubTypes.Type(value = HDFSCredentials::class, name = HDFSCredentials.type), ) open class StorageCredentials( open var key: String? = null, open var cache: CacheProperties, open var upload: UploadProperties, open var encrypt: EncryptProperties, + open var compress: CompressProperties, ) diff --git a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/message/StorageMessageCode.kt b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/message/StorageMessageCode.kt index a6e9181701..aaebc1b2c5 100644 --- a/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/message/StorageMessageCode.kt +++ b/src/backend/common/common-storage/storage-api/src/main/kotlin/com/tencent/bkrepo/common/storage/message/StorageMessageCode.kt @@ -44,7 +44,10 @@ enum class StorageMessageCode(private val key: String) : MessageCode { QUERY_ERROR("storage.query.error"), COPY_ERROR("storage.copy.error"), BLOCK_EMPTY("storage.block.empty"), - BLOCK_MISSING("storage.block.missing"); + BLOCK_MISSING("storage.block.missing"), + COMPRESS_ERROR("storage.compress.error"), + RESTORE_ERROR("storage.restore.error"), + ; override fun getBusinessCode() = ordinal + 1 override fun getKey() = key diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractEncryptorFileStorage.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractEncryptorFileStorage.kt index 0016b08a92..378526edbd 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractEncryptorFileStorage.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractEncryptorFileStorage.kt @@ -66,7 +66,7 @@ abstract class AbstractEncryptorFileStorage : F name: String, inputStream: InputStream, size: Long, - storageCredentials: StorageCredentials + storageCredentials: StorageCredentials, ) { val client = getClient(storageCredentials) retryTemplate.execute { @@ -124,7 +124,7 @@ abstract class AbstractFileStorage : F path: String, name: String, range: Range, - storageCredentials: StorageCredentials + storageCredentials: StorageCredentials, ): InputStream? { val client = getClient(storageCredentials) return try { @@ -152,7 +152,7 @@ abstract class AbstractFileStorage : F path: String, name: String, fromCredentials: StorageCredentials, - toCredentials: StorageCredentials + toCredentials: StorageCredentials, ) { val fromClient = getClient(fromCredentials) val toClient = getClient(toCredentials) diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractStorageService.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractStorageService.kt index cac73cebde..f47bd83c53 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractStorageService.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/AbstractStorageService.kt @@ -47,13 +47,13 @@ import kotlin.system.measureNanoTime * 存储服务抽象实现 */ @Suppress("TooGenericExceptionCaught") -abstract class AbstractStorageService : OverlaySupport() { +abstract class AbstractStorageService : CompressSupport() { override fun store( digest: String, artifactFile: ArtifactFile, storageCredentials: StorageCredentials?, - cancel: AtomicBoolean? + cancel: AtomicBoolean?, ): Int { val path = fileLocator.locate(digest) val credentials = getCredentialsOrDefault(storageCredentials) diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/CompressSupport.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/CompressSupport.kt new file mode 100644 index 0000000000..e63bdf0493 --- /dev/null +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/CompressSupport.kt @@ -0,0 +1,212 @@ +package com.tencent.bkrepo.common.storage.core + +import com.google.common.cache.CacheBuilder +import com.google.common.cache.CacheLoader +import com.google.common.cache.LoadingCache +import com.google.common.cache.RemovalListener +import com.tencent.bkrepo.common.api.util.StreamUtils +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.bksync.BkSync +import com.tencent.bkrepo.common.bksync.file.BkSyncDeltaSource +import com.tencent.bkrepo.common.bksync.file.BDUtils +import com.tencent.bkrepo.common.bksync.transfer.exception.TooLowerReuseRateException +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.filesystem.FileSystemClient +import com.tencent.bkrepo.common.storage.message.StorageErrorException +import com.tencent.bkrepo.common.storage.message.StorageMessageCode +import com.tencent.bkrepo.common.storage.util.createFile +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.time.Duration +import org.slf4j.LoggerFactory + +/** + * 压缩操作实现类 + * */ +abstract class CompressSupport : OverlaySupport() { + + /** + * 签名文件缓存 + * */ + private val checksumFileCache: LoadingCache by lazy { + CacheBuilder.newBuilder() + .expireAfterAccess(Duration.ofHours(1)) + .maximumSize(1000) + .removalListener( + RemovalListener { it.value?.delete() }, + ) + .build(CacheLoader.from(this::signFile)) + } + + override fun compress( + digest: String, + base: String, + storageCredentials: StorageCredentials?, + keep: Boolean, + ): Long { + // 增量存储源文件和基础文件必须不同，不然会导致base文件丢失 + require(digest != base) { "Incremental storage source file and base file must be different." } + val credentials = getCredentialsOrDefault(storageCredentials) + val workDir = getWorkDir(digest, credentials) + + // 文件正在被压缩 + if (Files.exists(workDir)) { + logger.info("The file [$digest] is compressing.") + return -1 + } + + // 文件已经被压缩 + if (isCompressed(digest, storageCredentials)) { + val bdFileName = digest.plus(BD_FILE_SUFFIX) + val bdFilePath = fileLocator.locate(bdFileName) + fileStorage.load(bdFilePath, bdFileName, Range.FULL_RANGE, credentials)?.use { + return BkSyncDeltaSource.readHeader(it).size + } + } + // 压缩文件 + try { + /* + * 压缩过程 + * 1. 下载源文件 + * 2. 获取base签名文件 + * 3. 根据原文件和签名文件，进行BD压缩，产生bd压缩文件 + * 4. 存储bd压缩文件 + * 5. 删除原文件（可选，根据keep参数决定） + * */ + val originFile = download(digest, credentials, workDir) + val baseFileKey = FileKey(base, credentials) + // base file可能会被多个其他版本文件使用，所以这里对基文件的签名进行了缓存 + var checksumFile = checksumFileCache.get(baseFileKey) + if (!checksumFile.exists()) { + checksumFileCache.invalidate(baseFileKey) + checksumFile = checksumFileCache.get(baseFileKey) + } + val threshold = credentials.compress.ratio + val bdFile = BDUtils.deltaByChecksumFile(originFile, checksumFile, digest, base, workDir, threshold) + val newFileName = digest.plus(BD_FILE_SUFFIX) + val newFilePath = fileLocator.locate(newFileName) + fileStorage.store(newFilePath, newFileName, bdFile, credentials) + if (!keep) { + delete(digest, credentials) + } + logger.info("Success to compress file [$digest] on ${credentials.key}.") + return bdFile.length() + } catch (e: TooLowerReuseRateException) { + throw e + } catch (e: Exception) { + logger.error("Failed to compress file [$digest] on ${credentials.key}.", e) + throw StorageErrorException(StorageMessageCode.COMPRESS_ERROR) + } finally { + workDir.toFile().deleteRecursively() + } + } + + override fun uncompress(digest: String, storageCredentials: StorageCredentials?): Int { + val credentials = getCredentialsOrDefault(storageCredentials) + val path = fileLocator.locate(digest) + val workDir = getWorkDir(digest, credentials) + // 文件正在被解压或者不存在压缩文件，直接返回 + if (Files.exists(workDir) || !isCompressed(digest, storageCredentials)) { + logger.info("The file[$digest] does not exist or is being decompressed.") + return 0 + } + // 已有未解压的文件，删除压缩文件 + if (fileStorage.exist(path, digest, credentials)) { + deleteCompressed(digest, storageCredentials) + return 1 + } + // 解压文件 + try { + /* + * 解压过程 + * 1. 下载digest的bd压缩文件 + * 2. 读取base信息，并判断base是否也是压缩文件，如果是先解压base + * 3. 下载base原文件 + * 4. 根据bd压缩文件和base原文件，合并成digest原文件 + * 5. 存储digest原文件 + * 6. 删除压缩文件 + * */ + val bdFileName = digest.plus(BD_FILE_SUFFIX) + val bdFile = download(bdFileName, credentials, workDir) + val base = bdFile.inputStream().use { BkSyncDeltaSource.readHeader(it).dest } + if (isCompressed(base, credentials)) { + logger.info("Base file [$base] is a delta file too,restore it first.") + uncompress(base, credentials) + } + val baseFile = download(base, credentials, workDir) + val originFile = BDUtils.patch(bdFile, baseFile, workDir) + fileStorage.store(path, digest, originFile, credentials) + delete(bdFileName, storageCredentials) + logger.info("Success to restore $digest on ${credentials.key}") + return 1 + } catch (e: Exception) { + logger.error("Failed to restore file [$digest] on ${credentials.key}.", e) + throw StorageErrorException(StorageMessageCode.RESTORE_ERROR) + } finally { + workDir.toFile().deleteRecursively() + } + } + + override fun isCompressed(digest: String, storageCredentials: StorageCredentials?): Boolean { + val bdFileName = digest.plus(BD_FILE_SUFFIX) + return exist(bdFileName, storageCredentials) + } + + override fun deleteCompressed(digest: String, storageCredentials: StorageCredentials?) { + val bdFileName = digest.plus(BD_FILE_SUFFIX) + delete(bdFileName, storageCredentials) + } + + /** + * 获取压缩工作路径 + * */ + private fun getWorkDir(digest: String, storageCredentials: StorageCredentials?): Path { + return Paths.get(getTempPath(storageCredentials).toString(), COMPRESS_WORK_DIR, digest) + } + + /** + * 下载[digest]到指定目录[dir] + * */ + protected fun download(digest: String, credentials: StorageCredentials, dir: Path): File { + val tempFile = FileSystemClient(dir).touch("", "$digest.temp") + try { + val path = fileLocator.locate(digest) + val inputStream = fileStorage.load(path, digest, Range.FULL_RANGE, credentials) + ?: error("Miss data $digest on ${credentials.key}") + StreamUtils.useCopy(inputStream, tempFile.outputStream()) + return tempFile + } catch (e: Exception) { + tempFile.delete() + throw e + } + } + + /** + * 签名指定的文件 + * */ + private fun signFile(key: FileKey): File { + val file = download(key.digest, key.storageCredentials, getTempPath(key.storageCredentials)) + try { + val checksumFile = getTempPath(key.storageCredentials) + .resolve(key.digest.plus(SIGN_FILE_SUFFIX)).createFile() + checksumFile.outputStream().use { BkSync().checksum(file, it) } + return checksumFile + } finally { + file.delete() + } + } + + private data class FileKey( + val digest: String, + val storageCredentials: StorageCredentials, + ) + + companion object { + private val logger = LoggerFactory.getLogger(CompressSupport::class.java) + private const val COMPRESS_WORK_DIR = "compress" + private const val BD_FILE_SUFFIX = ".bd" + private const val SIGN_FILE_SUFFIX = ".checksum" + } +} diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/StorageService.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/StorageService.kt index 5cd3d8f70a..da78ca179c 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/StorageService.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/StorageService.kt @@ -35,6 +35,7 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.storage.core.operation.CleanupOperation +import com.tencent.bkrepo.common.storage.core.operation.CompressOperation import com.tencent.bkrepo.common.storage.core.operation.FileBlockOperation import com.tencent.bkrepo.common.storage.core.operation.HealthCheckOperation import com.tencent.bkrepo.common.storage.core.operation.OverlayOperation @@ -46,7 +47,12 @@ import java.util.concurrent.atomic.AtomicBoolean /** * 存储服务接口 */ -interface StorageService : OverlayOperation, FileBlockOperation, HealthCheckOperation, CleanupOperation { +interface StorageService : + CompressOperation, + OverlayOperation, + FileBlockOperation, + HealthCheckOperation, + CleanupOperation { /** * 在存储实例[storageCredentials]上存储摘要为[digest]的构件[artifactFile] * 返回文件影响数，如果文件已经存在则返回0，否则返回1 @@ -55,7 +61,7 @@ interface StorageService : OverlayOperation, FileBlockOperation, HealthCheckOper digest: String, artifactFile: ArtifactFile, storageCredentials: StorageCredentials?, - cancel: AtomicBoolean? = null + cancel: AtomicBoolean? = null, ): Int /** diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt index f5ef0c9820..28a045db75 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageService.kt @@ -219,7 +219,8 @@ class CacheStorageService( * 当cacheFirst开启，并且cache磁盘健康，并且当前文件未超过内存阈值大小 */ private fun isLoadCacheFirst(range: Range, credentials: StorageCredentials): Boolean { - val isExceedThreshold = range.total > storageProperties.receive.fileSizeThreshold.toBytes() + val total = range.total ?: return false + val isExceedThreshold = total > storageProperties.receive.fileSizeThreshold.toBytes() val isHealth = getMonitor(credentials).healthy.get() val cacheFirst = credentials.cache.loadCacheFirst return cacheFirst && isHealth && isExceedThreshold diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/operation/CompressOperation.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/operation/CompressOperation.kt new file mode 100644 index 0000000000..7d16e688d0 --- /dev/null +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/operation/CompressOperation.kt @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.storage.core.operation + +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials + +/** + * 存储压缩操作 + * */ +interface CompressOperation { + /** + * 压缩[digest] + * + * 使用BD压缩，压缩[digest]。[base]与[digest]越相似，压缩率越高。压缩后生成digest.bd压缩文件 + * @param keep 压缩后，源文件是否保留 + * @return 压缩后的文件大小，-1表示未进行压缩 + * */ + fun compress( + digest: String, + base: String, + storageCredentials: StorageCredentials?, + keep: Boolean = false, + ): Long + + /** + * 解压[digest],解压成功后，删除bd压缩文件 + * + * @return 返回1表示成功解压，否则返回0 + * */ + fun uncompress(digest: String, storageCredentials: StorageCredentials?): Int + + /** + * [digest]是否被压缩 + * @return true文件被压缩，false文件未被压缩 + * */ + fun isCompressed(digest: String, storageCredentials: StorageCredentials?): Boolean + + /** + * 删除[digest]压缩文件 + * */ + fun deleteCompressed(digest: String, storageCredentials: StorageCredentials?) +} diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/overlay/OverlayRangeUtils.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/overlay/OverlayRangeUtils.kt index 96b17f872c..07e45b9e13 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/overlay/OverlayRangeUtils.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/overlay/OverlayRangeUtils.kt @@ -40,12 +40,14 @@ import kotlin.math.max object OverlayRangeUtils { fun build(blocks: List, range: Range): List { - val initialBlock = RegionResource(RegionResource.ZERO_RESOURCE, 0, range.total, 0, range.total) + val total = range.total + require(total != null) + val initialBlock = RegionResource(RegionResource.ZERO_RESOURCE, 0, total, 0, total) val list = LinkedList() list.add(initialBlock) // 完整文件分布 blocks.forEach { - insert(it, list, range.total) + insert(it, list, total) } // 范围内的文件分布 val newList = LinkedList() @@ -61,7 +63,7 @@ object OverlayRangeUtils { it.pos, it.size, max(range.start - it.pos + it.off, it.off), - min(len1, len2) + min(len1, len2), ) newList.add(part) } @@ -83,7 +85,7 @@ object OverlayRangeUtils { block.endPos + 1, res.size, res.off + block.endPos - res.pos + 1, - res.endPos - block.endPos + res.endPos - block.endPos, ) list.removeAt(index) val needAdd = listOf(part1, part2, part3) @@ -116,7 +118,7 @@ object OverlayRangeUtils { block.endPos + 1, nextRes.size, nextRes.off + block.endPos - nextRes.pos + 1, - nextRes.endPos - block.endPos + nextRes.endPos - block.endPos, ) list.removeAt(index) list.removeAt(index) diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/innercos/InnerCosFileStorage.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/innercos/InnerCosFileStorage.kt index 4676dc804f..c65b8bb21f 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/innercos/InnerCosFileStorage.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/innercos/InnerCosFileStorage.kt @@ -57,7 +57,11 @@ open class InnerCosFileStorage : AbstractEncryptorFileStorage() { private var defaultTransferManager: TransferManager? = null @@ -71,7 +71,9 @@ class S3Storage( override fun load(path: String, name: String, range: Range, client: S3Client): InputStream? { val getObjectRequest = GetObjectRequest(client.bucketName, name) - getObjectRequest.setRange(range.start, range.end) + if (range.isPartialContent()) { + getObjectRequest.setRange(range.start, range.end) + } return client.s3Client.getObject(getObjectRequest).objectContent } diff --git a/src/backend/common/common-storage/storage-service/src/test/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageServiceTest.kt b/src/backend/common/common-storage/storage-service/src/test/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageServiceTest.kt index 58142640fb..5e187deb0d 100644 --- a/src/backend/common/common-storage/storage-service/src/test/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageServiceTest.kt +++ b/src/backend/common/common-storage/storage-service/src/test/kotlin/com/tencent/bkrepo/common/storage/core/cache/CacheStorageServiceTest.kt @@ -60,6 +60,8 @@ import java.nio.charset.Charset import java.util.concurrent.CyclicBarrier import java.util.concurrent.atomic.AtomicBoolean import kotlin.concurrent.thread +import kotlin.random.Random +import org.springframework.util.StreamUtils @ExtendWith(SpringExtension::class) @ImportAutoConfiguration(StorageAutoConfiguration::class, TaskExecutionAutoConfiguration::class) @@ -234,6 +236,104 @@ internal class CacheStorageServiceTest { assertDoesNotThrow { storageService.store(sha256, artifactFile, null, cancel) } } + @Test + fun deltaStoreTest() { + val data1 = Random.nextBytes(Random.nextInt(1024, 1 shl 20)) + val data2 = data1.copyOfRange(Random.nextInt(1, 10), data1.size) + val artifactFile1 = createTempArtifactFile(data1) + val artifactFile2 = createTempArtifactFile(data2) + val originFileSize = artifactFile1.getSize() + try { + val digest1 = artifactFile1.getFileSha256() + val digest2 = artifactFile2.getFileSha256() + println(artifactFile1.getFileMd5()) + storageService.store(digest1, artifactFile1, null) + storageService.store(digest2, artifactFile2, null) + + // 增量存储 + val compressedSize = storageService.compress( + digest1, + digest2, + null, + ) + // 压缩后，源文件已经不在 + Assertions.assertNull(storageService.load(digest1, Range.full(originFileSize), null)) + + // 确定压缩后，实际存储变小 + val compressFileName = digest1.plus(".bd") + val compressFilepath = fileLocator.locate(compressFileName) + fileStorage.load( + compressFilepath, + compressFileName, + Range.FULL_RANGE, + storageProperties.defaultStorageCredentials(), + ).use { + val compressedFileSize = StreamUtils.drain(it!!).toLong() + Assertions.assertEquals(compressedSize, compressedFileSize) + Assertions.assertTrue(compressedFileSize < originFileSize) + } + + // 恢复文件 + val restored = storageService.uncompress(digest1, null) + Assertions.assertEquals(1, restored) + // 恢复后，数据不变 + val newLoad = storageService.load(digest1, Range.full(originFileSize), null) + ?.use { it.readBytes() } + Assertions.assertArrayEquals(data1, newLoad) + } finally { + artifactFile1.delete() + artifactFile2.delete() + } + } + + @Test + fun cascadeDeltaStore() { + val data1 = Random.nextBytes(Random.nextInt(1024, 1 shl 20)) + val data2 = data1.copyOfRange(Random.nextInt(1, 10), data1.size) + val data3 = data1.copyOfRange(Random.nextInt(11, 100), Random.nextInt(data1.size - 10)) + val artifactFile1 = createTempArtifactFile(data1) + val artifactFile2 = createTempArtifactFile(data2) + val artifactFile3 = createTempArtifactFile(data3) + val file1Len = artifactFile1.getSize() + val file2Len = artifactFile2.getSize() + try { + val digest1 = artifactFile1.getFileSha256() + val digest2 = artifactFile2.getFileSha256() + val digest3 = artifactFile3.getFileSha256() + storageService.store(digest1, artifactFile1, null) + storageService.store(digest2, artifactFile2, null) + storageService.store(digest3, artifactFile3, null) + storageService.compress( + digest1, + digest2, + null, + ) + storageService.compress( + digest2, + digest3, + null, + ) + // 压缩后，源文件已经不在 + // 压缩后，源文件已经不在 + Assertions.assertNull(storageService.load(digest1, Range.full(file1Len), null)) + Assertions.assertNull(storageService.load(digest2, Range.full(file2Len), null)) + // 级联恢复 + val restored = storageService.uncompress(digest1, null) + Assertions.assertEquals(1, restored) + // 恢复后，数据不变 + val file1Data = storageService.load(digest1, Range.full(file1Len), null) + ?.use { it.readBytes() } + Assertions.assertArrayEquals(data1, file1Data) + val file2Data = storageService.load(digest2, Range.full(file1Len), null) + ?.use { it.readBytes() } + Assertions.assertArrayEquals(data2, file2Data) + } finally { + artifactFile1.delete() + artifactFile2.delete() + artifactFile3.delete() + } + } + private fun createTempArtifactFile(size: Long): ArtifactFile { val tempFile = createTempFile() val content = StringPool.randomString(size.toInt()) @@ -244,4 +344,10 @@ internal class CacheStorageServiceTest { } return FileSystemArtifactFile(tempFile) } + + private fun createTempArtifactFile(data: ByteArray): ArtifactFile { + val tempFile = createTempFile() + tempFile.writeBytes(data) + return FileSystemArtifactFile(tempFile) + } } diff --git a/src/backend/common/common-storage/storage-service/src/test/kotlin/com/tencent/bkrepo/common/storage/monitor/StorageHealthMonitorHelperTest.kt b/src/backend/common/common-storage/storage-service/src/test/kotlin/com/tencent/bkrepo/common/storage/monitor/StorageHealthMonitorHelperTest.kt index 63bebdd935..26aa2801e5 100644 --- a/src/backend/common/common-storage/storage-service/src/test/kotlin/com/tencent/bkrepo/common/storage/monitor/StorageHealthMonitorHelperTest.kt +++ b/src/backend/common/common-storage/storage-service/src/test/kotlin/com/tencent/bkrepo/common/storage/monitor/StorageHealthMonitorHelperTest.kt @@ -28,7 +28,8 @@ class StorageHealthMonitorHelperTest { val otherStorageCredentials = StorageCredentials( upload = UploadProperties(location = storageCredentials.upload.location.plus("/temp")), cache = storageCredentials.cache, - encrypt = storageCredentials.encrypt + encrypt = storageCredentials.encrypt, + compress = storageCredentials.compress, ) val m1 = monitorHelper.getMonitor(storageProperties, otherStorageCredentials) val m2 = monitorHelper.getMonitor(storageProperties, otherStorageCredentials) diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt index a4044d0be9..5509545d40 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/FileOperationsHandler.kt @@ -100,7 +100,7 @@ class FileOperationsHandler( artifactInputStream.range.length ).awaitSingleOrNull() } else { - val source = RegionInputStreamResource(artifactInputStream, range.total) + val source = RegionInputStreamResource(artifactInputStream, range.total!!) val body = DataBufferUtils.read(source, DefaultDataBufferFactory.sharedInstance, DEFAULT_BUFFER_SIZE) response.writeWith(body).awaitSingleOrNull() } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJob.kt index de250b9c63..08098e1348 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/FileReferenceCleanupJob.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.job.batch import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.request.ArchiveFileRequest +import com.tencent.bkrepo.archive.request.DeleteCompressRequest import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.common.storage.core.StorageService @@ -96,11 +97,11 @@ class FileReferenceCleanupJob( return } storageService.delete(sha256, storageCredentials) - deleteArchive(sha256, credentialsKey) } else { - (context as FileJobContext).fileMissing.incrementAndGet() + context.fileMissing.incrementAndGet() logger.warn("File[$sha256] is missing on [$storageCredentials], skip cleaning up.") } + cleanupRelatedResources(sha256, credentialsKey) mongoTemplate.remove(Query(Criteria(ID).isEqualTo(id)), collectionName) } catch (e: Exception) { throw JobExecuteException("Failed to delete file[$sha256] on [$storageCredentials].", e) @@ -130,13 +131,11 @@ class FileReferenceCleanupJob( } } - private fun deleteArchive(sha256: String, credentialsKey: String?) { - val deleteArchiveFileRequest = ArchiveFileRequest( - sha256 = sha256, - storageCredentialsKey = credentialsKey, - operator = SYSTEM_USER, - ) + private fun cleanupRelatedResources(sha256: String, credentialsKey: String?) { + val deleteArchiveFileRequest = ArchiveFileRequest(sha256, credentialsKey, SYSTEM_USER) archiveClient.delete(deleteArchiveFileRequest) + val deleteCompressRequest = DeleteCompressRequest(sha256, credentialsKey, SYSTEM_USER) + archiveClient.deleteCompress(deleteCompressRequest) } private val cacheMap: ConcurrentHashMap = ConcurrentHashMap() diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/NodeCompressedJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/NodeCompressedJob.kt new file mode 100644 index 0000000000..03c7de258e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/NodeCompressedJob.kt @@ -0,0 +1,103 @@ +package com.tencent.bkrepo.job.batch + +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.api.ArchiveClient +import com.tencent.bkrepo.archive.request.CompleteCompressRequest +import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob +import com.tencent.bkrepo.job.batch.context.NodeContext +import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils +import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils +import com.tencent.bkrepo.job.config.properties.NodeCompressedJobProperties +import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest +import java.time.Duration +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +/** + * 节点压缩任务 + * + * 将已压缩的node打赏compressed标签 + * 1. 找到所有已压缩的节点 + * 2. 设置compressed为true + * 3. 删除原文件 + * 4. 完成压缩 + * */ +@Component +@EnableConfigurationProperties(NodeCompressedJobProperties::class) +class NodeCompressedJob( + properties: NodeCompressedJobProperties, + val nodeClient: NodeClient, + val archiveClient: ArchiveClient, + val storageService: StorageService, +) : + MongoDbBatchJob(properties) { + override fun createJobContext(): NodeContext { + return NodeContext() + } + + override fun getLockAtMostFor(): Duration = Duration.ofDays(7) + + override fun collectionNames(): List { + return listOf("compress_file") + } + + override fun buildQuery(): Query { + return Query.query(Criteria.where("status").isEqualTo(CompressStatus.COMPRESSED)) + } + + override fun run(row: CompressFile, collectionName: String, context: NodeContext) { + with(row) { + listNode(sha256, storageCredentialsKey).forEach { + val compressedRequest = NodeCompressedRequest( + projectId = it.projectId, + repoName = it.repoName, + fullPath = it.fullPath, + operator = lastModifiedBy, + ) + nodeClient.compressedNode(compressedRequest) + } + val storageCredentials = storageCredentialsKey?.let { + RepositoryCommonUtils.getStorageCredentials(storageCredentialsKey) + } + storageService.delete(sha256, storageCredentials) + val request = CompleteCompressRequest(sha256, storageCredentialsKey, lastModifiedBy) + archiveClient.completeCompress(request) + } + } + + override fun mapToEntity(row: Map): CompressFile { + return CompressFile( + id = row[CompressFile::id.name].toString(), + storageCredentialsKey = row[CompressFile::storageCredentialsKey.name]?.toString(), + sha256 = row[CompressFile::sha256.name].toString(), + uncompressedSize = row[CompressFile::uncompressedSize.name].toString().toLong(), + lastModifiedBy = row[CompressFile::lastModifiedBy.name].toString(), + ) + } + + override fun entityClass(): Class { + return CompressFile::class.java + } + + data class CompressFile( + var id: String? = null, + val sha256: String, + val uncompressedSize: Long, + val storageCredentialsKey: String?, + val lastModifiedBy: String, + ) + + private fun listNode(sha256: String, storageCredentialsKey: String?): List { + val query = Query.query( + Criteria.where("sha256").isEqualTo(sha256) + .and("compressed").ne(true) + .and("deleted").isEqualTo(null), + ) + return NodeCommonUtils.findNodes(query, storageCredentialsKey) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/NodeUncompressedJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/NodeUncompressedJob.kt new file mode 100644 index 0000000000..204fa537e6 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/NodeUncompressedJob.kt @@ -0,0 +1,95 @@ +package com.tencent.bkrepo.job.batch + +import com.tencent.bkrepo.archive.CompressStatus +import com.tencent.bkrepo.archive.api.ArchiveClient +import com.tencent.bkrepo.archive.request.DeleteCompressRequest +import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob +import com.tencent.bkrepo.job.batch.context.NodeContext +import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils +import com.tencent.bkrepo.job.config.properties.NodeUncompressedJobProperties +import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest +import java.time.Duration +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +/** + * 节点解压任务 + * + * 将已解压的节点去掉compressed标签 + * 1. 找到所有已解压的node + * 2. 去掉compressed标签 + * 3. 删除压缩文件 + * */ +@Component +@EnableConfigurationProperties(NodeUncompressedJobProperties::class) +class NodeUncompressedJob( + properties: NodeUncompressedJobProperties, + val nodeClient: NodeClient, + val archiveClient: ArchiveClient, +) : + MongoDbBatchJob(properties) { + override fun createJobContext(): NodeContext { + return NodeContext() + } + + override fun getLockAtMostFor(): Duration = Duration.ofDays(7) + + override fun collectionNames(): List { + return listOf("compress_file") + } + + override fun buildQuery(): Query { + return Query.query(Criteria.where("status").isEqualTo(CompressStatus.UNCOMPRESSED)) + } + + override fun run(row: CompressFile, collectionName: String, context: NodeContext) { + with(row) { + listNode(sha256, storageCredentialsKey).forEach { + val compressedRequest = NodeUnCompressedRequest( + projectId = it.projectId, + repoName = it.repoName, + fullPath = it.fullPath, + operator = lastModifiedBy, + ) + nodeClient.uncompressedNode(compressedRequest) + } + val request = DeleteCompressRequest(sha256, storageCredentialsKey, lastModifiedBy) + archiveClient.deleteCompress(request) + } + } + + override fun mapToEntity(row: Map): CompressFile { + return CompressFile( + id = row[CompressFile::id.name].toString(), + storageCredentialsKey = row[CompressFile::storageCredentialsKey.name]?.toString(), + sha256 = row[CompressFile::sha256.name].toString(), + uncompressedSize = row[CompressFile::uncompressedSize.name].toString().toLong(), + lastModifiedBy = row[CompressFile::lastModifiedBy.name].toString(), + ) + } + + override fun entityClass(): Class { + return CompressFile::class.java + } + + data class CompressFile( + var id: String? = null, + val sha256: String, + val uncompressedSize: Long, + val storageCredentialsKey: String?, + val lastModifiedBy: String, + ) + + private fun listNode(sha256: String, storageCredentialsKey: String?): List { + val query = Query.query( + Criteria.where("sha256").isEqualTo(sha256) + .and("compressed").isEqualTo(true) + .and("deleted").isEqualTo(null), + ) + return NodeCommonUtils.findNodes(query, storageCredentialsKey) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/SystemGcJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/SystemGcJob.kt new file mode 100644 index 0000000000..f70fd6a88e --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/SystemGcJob.kt @@ -0,0 +1,170 @@ +package com.tencent.bkrepo.job.batch + +import com.tencent.bkrepo.archive.api.ArchiveClient +import com.tencent.bkrepo.archive.request.CompressFileRequest +import com.tencent.bkrepo.common.api.collection.groupBySimilar +import com.tencent.bkrepo.common.api.util.HumanReadable +import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID +import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils +import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 +import com.tencent.bkrepo.job.SHARDING_COUNT +import com.tencent.bkrepo.job.batch.base.DefaultContextJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils +import com.tencent.bkrepo.job.config.properties.SystemGcJobProperties +import org.apache.commons.text.similarity.HammingDistance +import java.time.Duration +import java.time.LocalDateTime +import kotlin.math.abs +import org.bson.types.ObjectId +import org.slf4j.LoggerFactory +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component +import kotlin.system.measureNanoTime + +/** + * 存储GC任务 + * 找到相似的节点，进行增量压缩，以减少不必要的存储。 + * */ +@Component +@EnableConfigurationProperties(SystemGcJobProperties::class) +class SystemGcJob( + val properties: SystemGcJobProperties, + private val mongoTemplate: MongoTemplate, + private val archiveClient: ArchiveClient, +) : DefaultContextJob(properties) { + + private var lastId = MIN_OBJECT_ID + override fun doStart0(jobContext: JobContext) { + properties.repos.forEach { + val splits = it.split("/") + var count: Long + val projectId = splits[0] + val repoName = splits[1] + val nanos = measureNanoTime { count = repoGc(projectId, repoName) } + logger.info("Finish gc repository [$projectId/$repoName]($count nodes), took ${HumanReadable.time(nanos)}.") + } + } + + private fun repoGc(projectId: String, repoName: String): Long { + lastId = MIN_OBJECT_ID + val seq = HashShardingUtils.shardingSequenceFor(projectId, SHARDING_COUNT) + val collectionName = "node_$seq" + var nodes = mongoTemplate.find(buildQuery(projectId, repoName), Node::class.java, collectionName) + var count: Long = 0 + while (nodes.size > properties.nodeLimit) { + logger.info("Find ${nodes.size} nodes.") + lastId = nodes.last().id + count += nodes.size + // 文件按类型与长度分类，降低聚合难度 + nodes.groupBy { it.name.substringAfterLast(".") + it.name.length } + .flatMap { this.groupAndMeasure(it.value) } + .filter { it.size > properties.retain } + .forEach { gc(it) } + nodes = mongoTemplate.find(buildQuery(projectId, repoName), Node::class.java, collectionName) + } + return count + } + + private fun groupAndMeasure(list: List): List> { + if (list.size == 1) { + return listOf(list) + } + logger.info("Counting nodes: ${list.size}.") + var groups: List> + val nanos = measureNanoTime { groups = list.groupBySimilar({ node -> node.name }, this::isSimilar) } + logger.info("Complete grouping,took ${HumanReadable.time(nanos)}.") + return groups + } + + fun isSimilar(node1: Node, node2: Node): Boolean { + // 大小差异过大 + if (abs(node1.size - node2.size) * 2 / (node1.size + node2.size) > 0.5) { + return false + } + val name1 = node1.name + val name2 = node2.name + val editDistance = HAMMING_DISTANCE_INSTANCE.apply(name1, name2) + val ratio = editDistance.toDouble() * 2 / (name1.length + name2.length) + if (logger.isDebugEnabled) { + logger.debug("ham($name1,$name2)=$editDistance ($ratio)") + } + return ratio < properties.edThreshold + } + + private fun buildQuery(projectId: String, repoName: String): Query { + val cutoffTime = LocalDateTime.now().minus(Duration.ofDays(properties.idleDays.toLong())) + return Query.query( + Criteria.where(ID).gt(ObjectId(lastId)) + .and("folder").isEqualTo(false) + .and("sha256").ne(FAKE_SHA256) + .and("deleted").isEqualTo(null) + .and("projectId").isEqualTo(projectId) + .and("repoName").isEqualTo(repoName) + .and("compressed").ne(true) // 未被压缩 + .and("archived").ne(true) // 未被归档 + .and("size").gt(properties.fileSizeThreshold.toBytes()) + .orOperator( + Criteria.where("lastAccessDate").isEqualTo(null), + Criteria.where("lastAccessDate").lt(cutoffTime), + ), + ).limit(properties.maxBatchSize).with(Sort.by(ID).ascending()) // 长时间未访问 + } + + /** + * 数据gc + * */ + private fun gc(nodes: List) { + val sortedNodes = nodes.distinctBy { it.sha256 } + .apply { + if ((size - properties.retain) < 1) { + return + } + } + .sortedBy { it.createdDate } + // 没有新的节点，表示节点已经gc过一轮了 + if (lastEndTime != null && sortedNodes.last().createdDate < lastBeginTime) { + logger.info("There are no new nodes, gc is skipped.") + return + } + val gcNodes = sortedNodes.subList(0, sortedNodes.size - properties.retain) + if (logger.isDebugEnabled) { + logger.debug("Group node: [${gcNodes.joinToString(",") { it.name }}]") + } + // 保留最新的 + val newest = sortedNodes.last() + val repo = RepositoryCommonUtils.getRepositoryDetail(newest.projectId, newest.repoName) + val credentials = repo.storageCredentials + gcNodes.forEach { + val compressedRequest = CompressFileRequest(it.sha256, it.size, newest.sha256, credentials?.key) + archiveClient.compress(compressedRequest) + logger.info("Compress node ${it.name} by node ${newest.name}.") + } + } + + data class Node( + val id: String, + val projectId: String, + val repoName: String, + val fullPath: String, + val sha256: String, + val size: Long, + val name: String, + val createdDate: LocalDateTime, + ) { + override fun toString(): String { + return "$projectId/$repoName$fullPath($sha256)" + } + } + + companion object { + private val logger = LoggerFactory.getLogger(SystemGcJob::class.java) + private val HAMMING_DISTANCE_INSTANCE = HammingDistance() + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt index ee2d33e681..bb2cccafaa 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt @@ -1,5 +1,6 @@ package com.tencent.bkrepo.job.batch.utils +import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.job.SHARDING_COUNT import java.time.LocalDateTime import org.springframework.data.mongodb.core.MongoTemplate @@ -39,5 +40,18 @@ class NodeCommonUtils( } return nodes } + + fun collectionNames(projectIds: List): List { + val collectionNames = mutableListOf() + if (projectIds.isNotEmpty()) { + projectIds.forEach { + val index = HashShardingUtils.shardingSequenceFor(it, SHARDING_COUNT) + collectionNames.add("${COLLECTION_NAME_PREFIX}$index") + } + } else { + (0 until SHARDING_COUNT).forEach { collectionNames.add("${COLLECTION_NAME_PREFIX}$it") } + } + return collectionNames + } } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/NodeCompressedJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/NodeCompressedJobProperties.kt new file mode 100644 index 0000000000..645decd423 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/NodeCompressedJobProperties.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(value = "job.node-compress") +class NodeCompressedJobProperties( + override var cron: String = "0 0 0/1 * * ?", +) : MongodbJobProperties() diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/NodeUncompressedJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/NodeUncompressedJobProperties.kt new file mode 100644 index 0000000000..f3e469ec01 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/NodeUncompressedJobProperties.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(value = "job.node-uncompress") +class NodeUncompressedJobProperties( + override var cron: String = "0 0 0/1 * * ?", +) : MongodbJobProperties() diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/SystemGcJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/SystemGcJobProperties.kt new file mode 100644 index 0000000000..55362e7d2f --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/SystemGcJobProperties.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.util.unit.DataSize + +@ConfigurationProperties(value = "job.system-gc") +data class SystemGcJobProperties( + override var cron: String = "0 0 0 * * ?", + /** + * 处理文件大小阈值 + * */ + var fileSizeThreshold: DataSize = DataSize.ofMegabytes(100), + /** + * 仓库信息 + * project/repo + * */ + var repos: Set = emptySet(), + /** + * 编辑距离阈值 + * */ + var edThreshold: Double = 0.3, + /** + * 保留最新次数 + * */ + var retain: Int = 3, + + /** + * 访问时间限制 + * */ + var idleDays: Int = 30, + + /** + * 最大批处理数量 + * */ + var maxBatchSize: Int = 100000, + + /** + * 只有超过该节点数量，才会进行gc + * */ + var nodeLimit: Int = 1000, +) : MongodbJobProperties() diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt index e011402d76..98914b6000 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt @@ -43,11 +43,13 @@ import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeCleanRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest @@ -79,7 +81,7 @@ interface NodeClient { fun getNodeDetail( @PathVariable projectId: String, @PathVariable repoName: String, - @RequestParam fullPath: String + @RequestParam fullPath: String, ): Response @ApiOperation("根据路径查看节点是否存在") @@ -87,7 +89,7 @@ interface NodeClient { fun checkExist( @PathVariable projectId: String, @PathVariable repoName: String, - @RequestParam fullPath: String + @RequestParam fullPath: String, ): Response @ApiOperation("列出仓库中已存在的节点") @@ -95,7 +97,7 @@ interface NodeClient { fun listExistFullPath( @PathVariable projectId: String, @PathVariable repoName: String, - @RequestBody fullPathList: List + @RequestBody fullPathList: List, ): Response> @PostMapping("/page/{projectId}/{repoName}") @@ -103,7 +105,7 @@ interface NodeClient { @PathVariable projectId: String, @PathVariable repoName: String, @RequestParam path: String, - @RequestBody option: NodeListOption = NodeListOption() + @RequestBody option: NodeListOption = NodeListOption(), ): Response> @ApiOperation("创建节点") @@ -146,24 +148,31 @@ interface NodeClient { @GetMapping("/size/{projectId}/{repoName}") fun computeSize( @ApiParam(value = "所属项目", required = true) - @PathVariable projectId: String, + @PathVariable + projectId: String, @ApiParam(value = "仓库名称", required = true) - @PathVariable repoName: String, + @PathVariable + repoName: String, @ApiParam(value = "节点完整路径", required = true) - @RequestParam fullPath: String, + @RequestParam + fullPath: String, @ApiParam(value = "估计值", required = false) - @RequestParam estimated: Boolean = false + @RequestParam + estimated: Boolean = false, ): Response @ApiOperation("查询文件节点数量") @GetMapping("/file/{projectId}/{repoName}") fun countFileNode( @ApiParam(value = "所属项目", required = true) - @PathVariable projectId: String, + @PathVariable + projectId: String, @ApiParam(value = "仓库名称", required = true) - @PathVariable repoName: String, + @PathVariable + repoName: String, @ApiParam(value = "节点完整路径", required = true) - @RequestParam path: String + @RequestParam + path: String, ): Response @ApiOperation("自定义查询节点，如不关注总记录数请使用queryWithoutCount") @@ -179,17 +188,23 @@ interface NodeClient { @GetMapping("/list/{projectId}/{repoName}") fun listNode( @ApiParam(value = "所属项目", required = true) - @PathVariable projectId: String, + @PathVariable + projectId: String, @ApiParam(value = "仓库名称", required = true) - @PathVariable repoName: String, + @PathVariable + repoName: String, @ApiParam(value = "所属目录", required = true) - @RequestParam path: String, + @RequestParam + path: String, @ApiParam(value = "是否包含目录", required = false, defaultValue = "true") - @RequestParam includeFolder: Boolean = true, + @RequestParam + includeFolder: Boolean = true, @ApiParam(value = "是否深度查询文件", required = false, defaultValue = "false") - @RequestParam deep: Boolean = false, + @RequestParam + deep: Boolean = false, @ApiParam(value = "是否包含元数据", required = false, defaultValue = "false") - @RequestParam includeMetadata: Boolean = false + @RequestParam + includeMetadata: Boolean = false, ): Response> @ApiOperation("查询已删除节点") @@ -197,7 +212,7 @@ interface NodeClient { fun getDeletedNodeDetail( @PathVariable projectId: String, @PathVariable repoName: String, - @RequestParam fullPath: String + @RequestParam fullPath: String, ): Response> @ApiOperation("通过sha256查询已删除节点") @@ -205,7 +220,7 @@ interface NodeClient { fun getDeletedNodeDetailBySha256( @PathVariable projectId: String, @PathVariable repoName: String, - @RequestParam sha256: String + @RequestParam sha256: String, ): Response /** @@ -222,6 +237,19 @@ interface NodeClient { @PutMapping("/archive/restore/") fun restoreNode(@RequestBody nodeArchiveRequest: NodeArchiveRequest): Response + /** + * 归档文件成功通知 + * */ + @ApiOperation("压缩节点") + @PutMapping("/compress/") + fun compressedNode(@RequestBody nodeCompressedRequest: NodeCompressedRequest): Response + + /** + * 恢复文件成功通知 + * */ + @ApiOperation("解压节点") + @PutMapping("/uncompress/") + fun uncompressedNode(@RequestBody nodeUnCompressedRequest: NodeUnCompressedRequest): Response @ApiOperation("清理最后修改时间早于{date}的文件节点") @DeleteMapping("/clean") diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeDetail.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeDetail.kt index ac25d89dde..396e39d038 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeDetail.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeDetail.kt @@ -85,6 +85,8 @@ data class NodeDetail( val clusterNames: Set? = nodeInfo.clusterNames, @ApiModelProperty("是否归档") val archived: Boolean? = nodeInfo.archived, + @ApiModelProperty("是否压缩") + val compressed: Boolean? = nodeInfo.compressed, ) { /** * 获取node所属package的name diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeInfo.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeInfo.kt index 4ff1577a57..be1742865e 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeInfo.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/NodeInfo.kt @@ -88,4 +88,6 @@ data class NodeInfo( val clusterNames: Set? = null, @ApiModelProperty("是否归档") val archived: Boolean? = null, + @ApiModelProperty("是否压缩") + val compressed: Boolean? = null, ) diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeCompressedRequest.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeCompressedRequest.kt new file mode 100644 index 0000000000..945edafbf6 --- /dev/null +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeCompressedRequest.kt @@ -0,0 +1,16 @@ +package com.tencent.bkrepo.repository.pojo.node.service + +import com.tencent.bkrepo.repository.pojo.ServiceRequest +import com.tencent.bkrepo.repository.pojo.node.NodeRequest +import io.swagger.annotations.ApiModelProperty + +data class NodeCompressedRequest( + @ApiModelProperty("所属项目", required = true) + override val projectId: String, + @ApiModelProperty("仓库名称", required = true) + override val repoName: String, + @ApiModelProperty("节点完整路径", required = true) + override val fullPath: String, + @ApiModelProperty("操作用户", required = true) + override val operator: String, +) : NodeRequest, ServiceRequest diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeUnCompressedRequest.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeUnCompressedRequest.kt new file mode 100644 index 0000000000..8b85171c92 --- /dev/null +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/node/service/NodeUnCompressedRequest.kt @@ -0,0 +1,16 @@ +package com.tencent.bkrepo.repository.pojo.node.service + +import com.tencent.bkrepo.repository.pojo.ServiceRequest +import com.tencent.bkrepo.repository.pojo.node.NodeRequest +import io.swagger.annotations.ApiModelProperty + +data class NodeUnCompressedRequest( + @ApiModelProperty("所属项目", required = true) + override val projectId: String, + @ApiModelProperty("仓库名称", required = true) + override val repoName: String, + @ApiModelProperty("节点完整路径", required = true) + override val fullPath: String, + @ApiModelProperty("操作用户", required = true) + override val operator: String, +) : NodeRequest, ServiceRequest diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt index 8ada9cfdd9..22d37acbec 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt @@ -45,11 +45,13 @@ import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeCleanRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest @@ -143,7 +145,7 @@ class NodeController( projectId: String, repoName: String, fullPath: String, - estimated: Boolean + estimated: Boolean, ): Response { val artifactInfo = DefaultArtifactInfo(projectId, repoName, fullPath) return ResponseBuilder.success(nodeService.computeSize(artifactInfo, estimated)) @@ -202,17 +204,29 @@ class NodeController( } override fun cleanNodes(nodeCleanRequest: NodeCleanRequest): Response { - return ResponseBuilder.success(nodeService.deleteBeforeDate( - projectId = nodeCleanRequest.projectId, - repoName = nodeCleanRequest.repoName, - path = nodeCleanRequest.path, - date = nodeCleanRequest.date, - operator = nodeCleanRequest.operator - )) + return ResponseBuilder.success( + nodeService.deleteBeforeDate( + projectId = nodeCleanRequest.projectId, + repoName = nodeCleanRequest.repoName, + path = nodeCleanRequest.path, + date = nodeCleanRequest.date, + operator = nodeCleanRequest.operator, + ), + ) } override fun restoreNode(nodeArchiveRequest: NodeArchiveRequest): Response { nodeService.restoreNode(nodeArchiveRequest) return ResponseBuilder.success() } + + override fun compressedNode(nodeCompressedRequest: NodeCompressedRequest): Response { + nodeService.compressedNode(nodeCompressedRequest) + return ResponseBuilder.success() + } + + override fun uncompressedNode(nodeUnCompressedRequest: NodeUnCompressedRequest): Response { + nodeService.uncompressedNode(nodeUnCompressedRequest) + return ResponseBuilder.success() + } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt index 32bfc47bfd..167abbb3f0 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt @@ -88,10 +88,11 @@ data class TNode( var clusterNames: Set? = null, var nodeNum: Long? = null, var archived: Boolean? = null, + var compressed: Boolean? = null, @ShardingKey(count = SHARDING_COUNT) var projectId: String, - var repoName: String + var repoName: String, ) { companion object { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeCompressOperation.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeCompressOperation.kt new file mode 100644 index 0000000000..9fd0b4c9e2 --- /dev/null +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeCompressOperation.kt @@ -0,0 +1,19 @@ +package com.tencent.bkrepo.repository.service.node + +import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest + +/** + * 节点压缩操作 + * */ +interface NodeCompressOperation { + /** + * 压缩节点 + * */ + fun compressedNode(nodeCompressedRequest: NodeCompressedRequest) + + /** + * 解压节点 + * */ + fun uncompressedNode(nodeUnCompressedRequest: NodeUnCompressedRequest) +} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeService.kt index 9261ef454e..abc9e16e8c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeService.kt @@ -41,4 +41,5 @@ interface NodeService : NodeMoveCopyOperation, NodeRenameOperation, NodeRestoreOperation, - NodeArchiveOperation + NodeArchiveOperation, + NodeCompressOperation diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt index 87d9f6fcb1..66179aa245 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt @@ -125,7 +125,7 @@ abstract class NodeBaseService( return Pages.ofResponse( Pages.ofRequest(option.pageNumber, option.pageSize), nodes.totalElements, - nodes.content.map { convert(it)!! } + nodes.content.map { convert(it)!! }, ) } @@ -162,7 +162,7 @@ abstract class NodeBaseService( } } - open fun buildTNode(request: NodeCreateRequest):TNode { + open fun buildTNode(request: NodeCreateRequest): TNode { with(request) { val normalizeFullPath = PathUtils.normalizeFullPath(fullPath) return TNode( @@ -179,16 +179,15 @@ abstract class NodeBaseService( nodeNum = null, metadata = MetadataUtils.compatibleConvertAndCheck( metadata, - MetadataUtils.changeSystem(nodeMetadata, repositoryProperties.allowUserAddSystemMetadata) + MetadataUtils.changeSystem(nodeMetadata, repositoryProperties.allowUserAddSystemMetadata), ), createdBy = createdBy ?: operator, createdDate = createdDate ?: LocalDateTime.now(), lastModifiedBy = createdBy ?: operator, lastModifiedDate = lastModifiedDate ?: LocalDateTime.now(), - lastAccessDate = LocalDateTime.now() + lastAccessDate = LocalDateTime.now(), ) } - } private fun afterCreate( @@ -196,7 +195,7 @@ abstract class NodeBaseService( node: TNode, createStart: Long, parents: List, - deletedTime: LocalDateTime? + deletedTime: LocalDateTime?, ) { with(node) { val createEnd = System.currentTimeMillis() @@ -228,8 +227,8 @@ abstract class NodeBaseService( nodeDao.remove( Query( Criteria(ID).isEqualTo(newNode.id) - .and(TNode::projectId).isEqualTo(projectId) - ) + .and(TNode::projectId).isEqualTo(projectId), + ), ) fileReferenceService.decrement(newNode) quotaService.decreaseUsedVolume(projectId, repoName, newNode.size) @@ -247,8 +246,8 @@ abstract class NodeBaseService( nodeDao.remove( Query( Criteria(ID).isEqualTo(dir.id) - .and(TNode::projectId).isEqualTo(dir.projectId) - ) + .and(TNode::projectId).isEqualTo(dir.projectId), + ), ) logger.info("Rollback node [$projectId/$repoName${dir.fullPath}]") } @@ -262,7 +261,7 @@ abstract class NodeBaseService( rootFullPath = fullPath, deletedTime = it, conflictStrategy = ConflictStrategy.FAILED, - operator = createdBy + operator = createdBy, ) restoreNode(restoreContext) logger.info("Restore node [$projectId/$repoName$fullPath]") @@ -367,7 +366,7 @@ abstract class NodeBaseService( createdBy = createdBy, createdDate = LocalDateTime.now(), lastModifiedBy = createdBy, - lastModifiedDate = LocalDateTime.now() + lastModifiedDate = LocalDateTime.now(), ) doCreate(node) nodes.addAll(creates) @@ -399,11 +398,11 @@ abstract class NodeBaseService( private fun checkNodeListOption(option: NodeListOption) { Preconditions.checkArgument( option.sortProperty.none { !TNode::class.java.declaredFields.map { f -> f.name }.contains(it) }, - "sortProperty" + "sortProperty", ) Preconditions.checkArgument( option.direction.none { it != Sort.Direction.DESC.name && it != Sort.Direction.ASC.name }, - "direction" + "direction", ) } @@ -439,7 +438,8 @@ abstract class NodeBaseService( deleted = it.deleted?.format(DateTimeFormatter.ISO_DATE_TIME), lastAccessDate = it.lastAccessDate?.format(DateTimeFormatter.ISO_DATE_TIME), clusterNames = it.clusterNames, - archived = it.archived + archived = it.archived, + compressed = it.compressed, ) } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeCompressSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeCompressSupport.kt new file mode 100644 index 0000000000..e012757981 --- /dev/null +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeCompressSupport.kt @@ -0,0 +1,37 @@ +package com.tencent.bkrepo.repository.service.node.impl + +import com.tencent.bkrepo.repository.dao.NodeDao +import com.tencent.bkrepo.repository.model.TNode +import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest +import com.tencent.bkrepo.repository.service.node.NodeCompressOperation +import com.tencent.bkrepo.repository.util.NodeQueryHelper +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.query.Update + +class NodeCompressSupport( + nodeBaseService: NodeBaseService, +) : NodeCompressOperation { + val nodeDao: NodeDao = nodeBaseService.nodeDao + override fun compressedNode(nodeCompressedRequest: NodeCompressedRequest) { + with(nodeCompressedRequest) { + val query = NodeQueryHelper.nodeQuery(projectId, repoName, fullPath) + val update = Update().set(TNode::compressed.name, true) + nodeDao.updateFirst(query, update) + logger.info("Success to compress node $projectId/$repoName/$fullPath") + } + } + + override fun uncompressedNode(nodeUnCompressedRequest: NodeUnCompressedRequest) { + with(nodeUnCompressedRequest) { + val query = NodeQueryHelper.nodeQuery(projectId, repoName, fullPath) + val update = Update().unset(TNode::compressed.name) + nodeDao.updateFirst(query, update) + logger.info("Success to uncompress node $projectId/$repoName/$fullPath") + } + } + + companion object { + private val logger = LoggerFactory.getLogger(NodeCompressSupport::class.java) + } +} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt index 8eb4ca731b..5ba6b53654 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt @@ -41,9 +41,11 @@ import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest import com.tencent.bkrepo.repository.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.repo.QuotaService @@ -64,7 +66,7 @@ class NodeServiceImpl( override val storageService: StorageService, override val quotaService: QuotaService, override val repositoryProperties: RepositoryProperties, - override val messageSupplier: MessageSupplier + override val messageSupplier: MessageSupplier, ) : NodeBaseService( nodeDao, repositoryDao, @@ -73,7 +75,7 @@ class NodeServiceImpl( storageService, quotaService, repositoryProperties, - messageSupplier + messageSupplier, ) { override fun computeSize(artifact: ArtifactInfo, estimated: Boolean): NodeSizeInfo { @@ -107,7 +109,7 @@ class NodeServiceImpl( projectId: String, repoName: String, fullPath: String, - operator: String + operator: String, ): NodeDeleteResult { return NodeDeleteSupport(this).deleteByPath(projectId, repoName, fullPath, operator) } @@ -117,7 +119,7 @@ class NodeServiceImpl( projectId: String, repoName: String, fullPaths: List, - operator: String + operator: String, ): NodeDeleteResult { return NodeDeleteSupport(this).deleteByPaths(projectId, repoName, fullPaths, operator) } @@ -127,7 +129,7 @@ class NodeServiceImpl( repoName: String, date: LocalDateTime, operator: String, - path: String + path: String, ): NodeDeleteResult { return NodeDeleteSupport(this).deleteBeforeDate(projectId, repoName, date, operator, path) } @@ -153,7 +155,9 @@ class NodeServiceImpl( override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { return NodeRestoreSupport(this).getDeletedNodeDetailBySha256( - projectId, repoName, sha256 + projectId, + repoName, + sha256, ) } @@ -177,4 +181,12 @@ class NodeServiceImpl( override fun restoreNode(nodeArchiveRequest: NodeArchiveRequest) { return NodeArchiveSupport(this).restoreNode(nodeArchiveRequest) } + + override fun compressedNode(nodeCompressedRequest: NodeCompressedRequest) { + return NodeCompressSupport(this).compressedNode(nodeCompressedRequest) + } + + override fun uncompressedNode(nodeUnCompressedRequest: NodeUnCompressedRequest) { + return NodeCompressSupport(this).uncompressedNode(nodeUnCompressedRequest) + } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt index fe298c3a7a..c615b0eec5 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -42,13 +42,16 @@ import com.tencent.bkrepo.repository.pojo.node.NodeRestoreOption import com.tencent.bkrepo.repository.pojo.node.NodeRestoreResult import com.tencent.bkrepo.repository.pojo.node.NodeSizeInfo import com.tencent.bkrepo.repository.pojo.node.service.NodeArchiveRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest import com.tencent.bkrepo.repository.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.impl.NodeArchiveSupport +import com.tencent.bkrepo.repository.service.node.impl.NodeCompressSupport import com.tencent.bkrepo.repository.service.node.impl.NodeDeleteSupport import com.tencent.bkrepo.repository.service.node.impl.NodeMoveCopySupport import com.tencent.bkrepo.repository.service.node.impl.NodeRenameSupport @@ -190,4 +193,12 @@ class EdgeNodeServiceImpl( override fun restoreNode(nodeArchiveRequest: NodeArchiveRequest) { return NodeArchiveSupport(this).restoreNode(nodeArchiveRequest) } + + override fun compressedNode(nodeCompressedRequest: NodeCompressedRequest) { + return NodeCompressSupport(this).compressedNode(nodeCompressedRequest) + } + + override fun uncompressedNode(nodeUnCompressedRequest: NodeUnCompressedRequest) { + return NodeCompressSupport(this).uncompressedNode(nodeUnCompressedRequest) + } }