From 2ae546facfa962c809e654f945457a63a28dc920 Mon Sep 17 00:00:00 2001 From: kunlongli <16629885+cnlkl@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:48:33 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=A4=9A=E7=A7=8D?= =?UTF-8?q?=E7=BB=B4=E5=BA=A6=E5=8F=96=E5=9B=9E=E9=A1=B9=E7=9B=AE=E5=BD=92?= =?UTF-8?q?=E6=A1=A3=E7=9A=84=E6=95=B0=E6=8D=AE=20#2219?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 支持多种维度取回项目归档的数据 #2219 * feat: 增加单元测试 #2219 * feat: 增加单元测试 #2219 --- .../bkrepo/job/pojo/ArchiveRestoreRequest.kt | 50 ++++++ .../controller/user/ArchiveJobController.kt | 6 +- .../bkrepo/job/service/ArchiveJobService.kt | 3 +- .../job/service/impl/ArchiveJobServiceImpl.kt | 44 ++++-- .../com/tencent/bkrepo/job/model/TNode.kt | 2 + .../service/impl/ArchiveJobServiceImplTest.kt | 142 ++++++++++++++++++ 6 files changed, 234 insertions(+), 13 deletions(-) create mode 100644 src/backend/job/api-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/ArchiveRestoreRequest.kt create mode 100644 src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImplTest.kt diff --git a/src/backend/job/api-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/ArchiveRestoreRequest.kt b/src/backend/job/api-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/ArchiveRestoreRequest.kt new file mode 100644 index 0000000000..62d3188353 --- /dev/null +++ b/src/backend/job/api-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/ArchiveRestoreRequest.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.pojo + +/** + * 恢复已归档或已压缩的制品请求 + */ +data class ArchiveRestoreRequest( + /** + * 制品所属项目 + */ + val projectId: String, + /** + * 制品所属仓库 + */ + val repoName: String? = null, + /** + * 制品路径前缀 + */ + val prefix: String? = null, + /** + * 制品元数据，所有元数据均匹配才会恢复 + */ + val metadata: Map = emptyMap(), +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/ArchiveJobController.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/ArchiveJobController.kt index 6a6fab81bb..95c702d12e 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/ArchiveJobController.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/controller/user/ArchiveJobController.kt @@ -3,8 +3,10 @@ package com.tencent.bkrepo.job.controller.user import com.tencent.bkrepo.archive.constant.ArchiveStorageClass import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType +import com.tencent.bkrepo.job.pojo.ArchiveRestoreRequest import com.tencent.bkrepo.job.service.ArchiveJobService import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController @@ -29,7 +31,7 @@ class ArchiveJobController( } @PostMapping("/restore") - fun restore(@RequestParam projectId: String) { - archiveJobService.restore(projectId) + fun restore(@RequestBody request: ArchiveRestoreRequest) { + archiveJobService.restore(request) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/ArchiveJobService.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/ArchiveJobService.kt index 49561c2855..4a6e18a81f 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/ArchiveJobService.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/ArchiveJobService.kt @@ -1,6 +1,7 @@ package com.tencent.bkrepo.job.service import com.tencent.bkrepo.archive.constant.ArchiveStorageClass +import com.tencent.bkrepo.job.pojo.ArchiveRestoreRequest /** * 归档任务服务 @@ -19,5 +20,5 @@ interface ArchiveJobService { /** * 恢复文件，从归档或者压缩中恢复文件 * */ - fun restore(projectId: String) + fun restore(request: ArchiveRestoreRequest) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt index ce97d5c20b..ec6378970d 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt @@ -5,6 +5,7 @@ import com.tencent.bkrepo.archive.constant.ArchiveStorageClass import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.archive.request.UncompressFileRequest import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils +import com.tencent.bkrepo.common.query.util.MongoEscapeUtils import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.BATCH_SIZE import com.tencent.bkrepo.job.SHARDING_COUNT @@ -14,8 +15,10 @@ import com.tencent.bkrepo.job.batch.task.archive.IdleNodeArchiveJob.Companion.CO import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService +import com.tencent.bkrepo.job.pojo.ArchiveRestoreRequest import com.tencent.bkrepo.job.service.ArchiveJobService import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -56,16 +59,9 @@ class ArchiveJobServiceImpl( } } - override fun restore(projectId: String) { - val query = Query.query( - Criteria.where("folder").isEqualTo(false) - .and("deleted").isEqualTo(null) - .and("sha256").ne(FAKE_SHA256) - .and("projectId").isEqualTo(projectId).orOperator( - Criteria.where("archived").isEqualTo(true), - Criteria.where("compressed").isEqualTo(true), - ), - ) + override fun restore(request: ArchiveRestoreRequest) { + val projectId = request.projectId + val query = Query(buildCriteria(request)) val index = HashShardingUtils.shardingSequenceFor(projectId, SHARDING_COUNT) val collectionName = COLLECTION_NAME_PREFIX.plus(index) val context = NodeContext() @@ -103,6 +99,34 @@ class ArchiveJobServiceImpl( } } + fun buildCriteria(request: ArchiveRestoreRequest): Criteria { + return with(request) { + val criteria = Criteria.where("folder").isEqualTo(false) + .and("deleted").isEqualTo(null) + .and("sha256").ne(FAKE_SHA256) + .and("projectId").isEqualTo(projectId).orOperator( + Criteria.where("archived").isEqualTo(true), + Criteria.where("compressed").isEqualTo(true), + ) + repoName?.let { criteria.and("repoName").isEqualTo(it) } + prefix?.let { criteria.and("fullPath").regex("^${MongoEscapeUtils.escapeRegex(it)}") } + val metadataCriteria = metadata.map { + val elemCriteria = Criteria().andOperator( + MetadataModel::key.isEqualTo(it.key), + MetadataModel::value.isEqualTo(it.value) + ) + Criteria.where("metadata").elemMatch(elemCriteria) + } + if (metadataCriteria.isEmpty()) { + criteria + } else { + val allCriteria = metadataCriteria.toMutableList() + allCriteria.add(criteria) + Criteria().andOperator(allCriteria) + } + } + } + companion object { private val logger = LoggerFactory.getLogger(ArchiveJobServiceImpl::class.java) } diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/model/TNode.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/model/TNode.kt index de2243423c..92aa84c7b1 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/model/TNode.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/model/TNode.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.job.model +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import org.springframework.data.mongodb.core.index.TextIndexDefinition.TextIndexDefinitionBuilder import java.time.LocalDateTime @@ -43,6 +44,7 @@ data class TNode( val deleted: LocalDateTime? = null, val archived: Boolean? = null, val compressed: Boolean? = null, + val metadata: List = emptyList(), ) { companion object { fun pathIndex() = TextIndexDefinitionBuilder() diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImplTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImplTest.kt new file mode 100644 index 0000000000..6d8a30c758 --- /dev/null +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImplTest.kt @@ -0,0 +1,142 @@ +package com.tencent.bkrepo.job.service.impl + +import com.tencent.bkrepo.archive.api.ArchiveClient +import com.tencent.bkrepo.job.UT_MD5 +import com.tencent.bkrepo.job.UT_PROJECT_ID +import com.tencent.bkrepo.job.UT_REPO_NAME +import com.tencent.bkrepo.job.UT_SHA256 +import com.tencent.bkrepo.job.batch.task.archive.IdleNodeArchiveJob +import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService +import com.tencent.bkrepo.job.model.TNode +import com.tencent.bkrepo.job.pojo.ArchiveRestoreRequest +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.context.annotation.Import +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Query +import java.time.LocalDateTime + +@DisplayName("归档任务服务测试") +@DataMongoTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Import(ArchiveJobServiceImpl::class) +class ArchiveJobServiceImplTest @Autowired constructor( + private val mongoTemplate: MongoTemplate, + private val service: ArchiveJobServiceImpl, +) { + @MockBean + private lateinit var archiveJob: IdleNodeArchiveJob + + @MockBean + private lateinit var archiveClient: ArchiveClient + + @MockBean + private lateinit var migrateRepoStorageService: MigrateRepoStorageService + + @Test + fun test() { + mockNode() + val req = ArchiveRestoreRequest( + projectId = UT_PROJECT_ID, + repoName = UT_REPO_NAME, + prefix = "/a", + ) + // 根据projectId筛选 + var nodes = findNodes(req) + assertEquals(1, nodes.size) + + nodes = findNodes(req.copy(projectId = "other")) + assertEquals(2, nodes.size) + + // 根据repo筛选 + nodes = findNodes(req.copy(repoName = "other")) + assertEquals(2, nodes.size) + + // 根据路径前缀筛选 + nodes = findNodes(req.copy(prefix = "/e/f/")) + assertEquals(2, nodes.size) + + // 根据元数据筛选 + nodes = findNodes( + req.copy( + projectId = "metadata-test", + metadata = mapOf( + "pid" to "ppp", + "bid" to "bbb" + ) + ) + ) + assertEquals(2, nodes.size) + } + + private fun findNodes(request: ArchiveRestoreRequest): List { + return mongoTemplate.find(Query(service.buildCriteria(request)), TNode::class.java, COLLECTION_NAME) + } + + private fun mockNode() { + val node = TNode( + id = null, + projectId = UT_PROJECT_ID, + repoName = UT_REPO_NAME, + fullPath = "/a/b/c.txt", + size = 100L, + sha256 = UT_SHA256, + md5 = UT_MD5, + createdDate = LocalDateTime.now(), + folder = false, + archived = true, + compressed = true, + ) + + mongoTemplate.insert(node, COLLECTION_NAME) + mongoTemplate.insert(node.copy(projectId = "other"), COLLECTION_NAME) + mongoTemplate.insert(node.copy(projectId = "other"), COLLECTION_NAME) + mongoTemplate.insert(node.copy(repoName = "other"), COLLECTION_NAME) + mongoTemplate.insert(node.copy(repoName = "other"), COLLECTION_NAME) + mongoTemplate.insert(node.copy(fullPath = "/e/f/g.txt"), COLLECTION_NAME) + mongoTemplate.insert(node.copy(fullPath = "/e/f/t.txt"), COLLECTION_NAME) + mongoTemplate.insert( + node.copy( + projectId = "metadata-test", + metadata = listOf( + MetadataModel(key = "pid", value = "ppp"), + MetadataModel(key = "bid", value = "bbb"), + MetadataModel(key = "xxx", value = "yyy"), + ) + ), + COLLECTION_NAME + ) + mongoTemplate.insert( + node.copy( + projectId = "metadata-test", + metadata = listOf( + MetadataModel(key = "pid", value = "ppp"), + MetadataModel(key = "bid", value = "bbb"), + MetadataModel(key = "xxx", value = "yyy"), + ) + ), + COLLECTION_NAME + ) + mongoTemplate.insert( + node.copy( + projectId = "metadata-test", + metadata = listOf( + MetadataModel(key = "pid", value = "ppp"), + MetadataModel(key = "bid", value = "bbb2"), + MetadataModel(key = "xxx", value = "yyy"), + ) + ), + COLLECTION_NAME + ) + } + + companion object { + private const val COLLECTION_NAME = "node_1" + } +}