增加系统GC功能 (TencentBlueKing#1557)

* feat: 支持系统GC和优化归档代码 TencentBlueKing#1483

* feat: 支持系统GC和优化归档代码 TencentBlueKing#1483

* feat: 修改metrics指标 TencentBlueKing#1483

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/ArchiveStatus.kt

@@ -16,6 +16,11 @@ enum class ArchiveStatus {
      * */
     ARCHIVED,
 
+    /**
+     * 已完成
+     * */
+    COMPLETED,
+
     /**
      * 待恢复
      * */
@@ -40,9 +45,4 @@ enum class ArchiveStatus {
      * 恢复失败
      * */
     RESTORE_FAILED,
-
-    /**
-     * 已完成
-     * */
-    COMPLETED,
 }

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/CompressStatus.kt

@@ -0,0 +1,53 @@
+package com.tencent.bkrepo.archive
+
+enum class CompressStatus {
+    /**
+     * 已创建
+     * */
+    CREATED,
+
+    /**
+     * 压缩中
+     * */
+    COMPRESSING,
+
+    /**
+     * 已压缩
+     * */
+    COMPRESSED,
+
+    /**
+     * 已完成
+     * */
+    COMPLETED,
+
+    /**
+     * 等待解压
+     * */
+    WAIT_TO_UNCOMPRESS,
+
+    /**
+     * 正在解压
+     * */
+    UNCOMPRESSING,
+
+    /**
+     * 已解压
+     * */
+    UNCOMPRESSED,
+
+    /**
+     * 表示链头
+     * */
+    NONE,
+
+    /**
+     * 压缩失败
+     * */
+    COMPRESS_FAILED,
+
+    /**
+     * 解压失败
+     * */
+    UNCOMPRESS_FAILED,
+}

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/api/ArchiveClient.kt

@@ -2,17 +2,25 @@ package com.tencent.bkrepo.archive.api
 
 import com.tencent.bkrepo.archive.pojo.ArchiveFile
 import com.tencent.bkrepo.archive.request.ArchiveFileRequest
+import com.tencent.bkrepo.archive.request.CompleteCompressRequest
+import com.tencent.bkrepo.archive.request.CompressFileRequest
 import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest
+import com.tencent.bkrepo.archive.request.DeleteCompressRequest
+import com.tencent.bkrepo.archive.request.UncompressFileRequest
 import com.tencent.bkrepo.common.api.constant.ARCHIVE_SERVICE_NAME
 import com.tencent.bkrepo.common.api.pojo.Response
 import org.springframework.cloud.openfeign.FeignClient
 import org.springframework.web.bind.annotation.DeleteMapping
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.PostMapping
+import org.springframework.web.bind.annotation.PutMapping
 import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.bind.annotation.RequestMapping
 import org.springframework.web.bind.annotation.RequestParam
 
+/**
+ * 归档服务接口
+ * */
 @FeignClient(ARCHIVE_SERVICE_NAME)
 @RequestMapping("/service/archive")
 interface ArchiveClient {
@@ -33,4 +41,16 @@ interface ArchiveClient {
         @RequestParam sha256: String,
         @RequestParam(required = false) storageCredentialsKey: String?,
     ): Response<ArchiveFile?>
+
+    @PutMapping("/compress")
+    fun compress(@RequestBody request: CompressFileRequest): Response<Void>
+
+    @PutMapping("/uncompress")
+    fun uncompress(@RequestBody request: UncompressFileRequest): Response<Void>
+
+    @DeleteMapping("/compress")
+    fun deleteCompress(@RequestBody request: DeleteCompressRequest): Response<Void>
+
+    @PutMapping("/compress/complete")
+    fun completeCompress(@RequestBody request: CompleteCompressRequest): Response<Void>
 }

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/constant/ArchiveConstants.kt

@@ -2,3 +2,4 @@ package com.tencent.bkrepo.archive.constant
 
 const val XZ_SUFFIX = ".xz"
 const val DEEP_ARCHIVE = "DEEP_ARCHIVE"
+const val DEFAULT_KEY = "default"

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/pojo/CompressedInfo.kt

@@ -0,0 +1,10 @@
+package com.tencent.bkrepo.archive.pojo
+
+import com.tencent.bkrepo.common.api.constant.StringPool
+
+class CompressedInfo(
+    val status: Int, // 0 压缩中， 1 压缩成功
+    val uncompressedSize: Long,
+    val compressedSize: Long,
+    val ratio: String = StringPool.calculateRatio(uncompressedSize, compressedSize),
+)

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompleteCompressRequest.kt

@@ -0,0 +1,9 @@
+package com.tencent.bkrepo.archive.request
+
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+
+data class CompleteCompressRequest(
+    val sha256: String,
+    val storageCredentialsKey: String?,
+    val operator: String = SYSTEM_USER,
+)

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/CompressFileRequest.kt

@@ -0,0 +1,11 @@
+package com.tencent.bkrepo.archive.request
+
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+
+data class CompressFileRequest(
+    val sha256: String,
+    val size: Long,
+    val baseSha256: String,
+    val storageCredentialsKey: String?,
+    val operator: String = SYSTEM_USER,
+)

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/DeleteCompressRequest.kt

@@ -0,0 +1,9 @@
+package com.tencent.bkrepo.archive.request
+
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+
+data class DeleteCompressRequest(
+    val sha256: String,
+    val storageCredentialsKey: String?,
+    val operator: String = SYSTEM_USER,
+)

src/backend/archive/api-archive/src/main/kotlin/com/tencent/bkrepo/archive/request/UncompressFileRequest.kt

@@ -0,0 +1,9 @@
+package com.tencent.bkrepo.archive.request
+
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
+
+data class UncompressFileRequest(
+    val sha256: String,
+    val storageCredentialsKey: String?,
+    val operator: String = SYSTEM_USER,
+)

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/config/ArchiveProperties.kt

@@ -5,6 +5,9 @@ import org.springframework.boot.context.properties.ConfigurationProperties
 import org.springframework.boot.context.properties.NestedConfigurationProperty
 import org.springframework.util.unit.DataSize
 
+/**
+ * 归档服务配置
+ * */
 @ConfigurationProperties("archive")
 data class ArchiveProperties(
     @NestedConfigurationProperty

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/controller/service/ArchiveController.kt

@@ -3,15 +3,24 @@ package com.tencent.bkrepo.archive.controller.service
 import com.tencent.bkrepo.archive.api.ArchiveClient
 import com.tencent.bkrepo.archive.pojo.ArchiveFile
 import com.tencent.bkrepo.archive.request.ArchiveFileRequest
+import com.tencent.bkrepo.archive.request.CompleteCompressRequest
+import com.tencent.bkrepo.archive.request.CompressFileRequest
 import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest
+import com.tencent.bkrepo.archive.request.DeleteCompressRequest
+import com.tencent.bkrepo.archive.request.UncompressFileRequest
 import com.tencent.bkrepo.archive.service.ArchiveService
+import com.tencent.bkrepo.archive.service.CompressService
 import com.tencent.bkrepo.common.api.pojo.Response
 import com.tencent.bkrepo.common.service.util.ResponseBuilder
 import org.springframework.web.bind.annotation.RestController
 
+/**
+ * 归档服务控制器
+ * */
 @RestController
 class ArchiveController(
     private val archiveService: ArchiveService,
+    private val compressService: CompressService,
 ) : ArchiveClient {
     override fun archive(request: CreateArchiveFileRequest): Response<Void> {
         archiveService.archive(request)
@@ -36,4 +45,24 @@ class ArchiveController(
         archiveService.complete(request)
         return ResponseBuilder.success()
     }
+
+    override fun compress(request: CompressFileRequest): Response<Void> {
+        compressService.compress(request)
+        return ResponseBuilder.success()
+    }
+
+    override fun uncompress(request: UncompressFileRequest): Response<Void> {
+        compressService.uncompress(request)
+        return ResponseBuilder.success()
+    }
+
+    override fun deleteCompress(request: DeleteCompressRequest): Response<Void> {
+        compressService.delete(request)
+        return ResponseBuilder.success()
+    }
+
+    override fun completeCompress(request: CompleteCompressRequest): Response<Void> {
+        compressService.complete(request)
+        return ResponseBuilder.success()
+    }
 }

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/FileCompressedEvent.kt

@@ -3,7 +3,7 @@ package com.tencent.bkrepo.archive.event
 import com.tencent.bkrepo.common.storage.monitor.Throughput
 
 /**
- * 文件归档事件
+ * 归档压缩事件
  * */
 data class FileCompressedEvent(
     val sha256: String,

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileCompressedEvent.kt

@@ -0,0 +1,15 @@
+package com.tencent.bkrepo.archive.event
+
+import com.tencent.bkrepo.common.storage.monitor.Throughput
+
+/**
+ * 存储文件压缩事件
+ * */
+data class StorageFileCompressedEvent(
+    val sha256: String,
+    val baseSha256: String,
+    val uncompressed: Long,
+    val compressed: Long,
+    val storageCredentialsKey: String?,
+    val throughput: Throughput,
+)

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/event/StorageFileUncompressedEvent.kt

@@ -0,0 +1,14 @@
+package com.tencent.bkrepo.archive.event
+
+import com.tencent.bkrepo.common.storage.monitor.Throughput
+
+/**
+ * 存储文件解压事件
+ * */
+data class StorageFileUncompressedEvent(
+    val sha256: String,
+    val uncompressed: Long,
+    val compressed: Long,
+    val storageCredentialsKey: String?,
+    val throughput: Throughput,
+)