From 98c59e8b0bd8da47d73b6cf12f91bfb2e2607933 Mon Sep 17 00:00:00 2001 From: kunlongli <16629885+cnlkl@users.noreply.github.com> Date: Tue, 28 Nov 2023 19:48:48 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E8=A7=A6=E5=8F=91?= =?UTF-8?q?=E5=85=A8=E5=B1=80=E6=89=AB=E6=8F=8F=E4=BB=BB=E5=8A=A1=20#877?= =?UTF-8?q?=20(#1301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../analyst/message/ScannerMessageCode.kt | 15 ++-- .../tencent/bkrepo/analyst/pojo/ScanTask.kt | 8 ++- .../bkrepo/analyst/pojo/TaskMetadata.kt | 5 ++ .../analyst/pojo/request/GlobalScanRequest.kt | 48 +++++++++++++ .../analyst/pojo/request/ScanTaskQuery.kt | 2 +- .../ScannerPermissionCheckHandler.kt | 5 ++ .../configuration/ScannerProperties.kt | 4 ++ .../controller/user/UserScanController.kt | 20 ++++++ .../tencent/bkrepo/analyst/dao/ScanTaskDao.kt | 24 +++++-- .../bkrepo/analyst/dao/SubScanTaskDao.kt | 5 +- .../tencent/bkrepo/analyst/model/TScanTask.kt | 6 +- .../ProjectScanConfigurationService.kt | 2 +- .../bkrepo/analyst/service/ScanService.kt | 12 +++- .../ProjectScanConfigurationServiceImpl.kt | 7 +- .../analyst/service/impl/ScanServiceImpl.kt | 47 ++++++++++++- .../service/impl/ScanTaskServiceImpl.kt | 6 +- .../statemachine/iterator/NodeIterator.kt | 12 +++- .../statemachine/task/action/PendingAction.kt | 34 +++++---- .../task/action/SubmittingAction.kt | 15 +++- .../tencent/bkrepo/analyst/utils/Converter.kt | 1 + .../bkrepo/analyst/utils/RuleConverter.kt | 20 +++--- .../resources/i18n/messages_en.properties | 1 + .../resources/i18n/messages_zh_CN.properties | 1 + .../resources/i18n/messages_zh_TW.properties | 1 + .../pojo/project/ProjectRangeQueryRequest.kt | 4 +- .../service/repo/impl/ProjectServiceImpl.kt | 28 +++++--- .../repository/service/ProjectServiceTest.kt | 69 ++++++++++++++++++- 27 files changed, 338 insertions(+), 64 deletions(-) create mode 100644 src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/request/GlobalScanRequest.kt diff --git a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/message/ScannerMessageCode.kt b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/message/ScannerMessageCode.kt index 8366c568ce..f1594fda2a 100644 --- a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/message/ScannerMessageCode.kt +++ b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/message/ScannerMessageCode.kt @@ -41,15 +41,15 @@ enum class ScannerMessageCode( SCAN_TASK_COUNT_EXCEED_LIMIT("scanner.task.count.exceed-limit", 5), SCAN_TASK_NAME_BATCH_SCAN("scanner.task.name.manual", 6), SCAN_TASK_NAME_SINGLE_SCAN("scanner.task.name.manual.single", 7), - SCAN_REPORT_NOTIFY_MESSAGE_SCANNED("scanner.report.notify.message.scanned",8), - SCAN_REPORT_NOTIFY_MESSAGE_CVE("scanner.report.notify.message.cve",10), - SCAN_REPORT_NOTIFY_MESSAGE_DETAIL("scanner.report.notify.message.detail",14), + SCAN_REPORT_NOTIFY_MESSAGE_SCANNED("scanner.report.notify.message.scanned", 8), + SCAN_REPORT_NOTIFY_MESSAGE_CVE("scanner.report.notify.message.cve", 10), + SCAN_REPORT_NOTIFY_MESSAGE_DETAIL("scanner.report.notify.message.detail", 14), SCAN_REPORT_NOTIFY_MESSAGE_TITLE("scanner.report.notify.message.title", 15), SCAN_REPORT_NOTIFY_MESSAGE_TRIGGER_TIME("scanner.report.notify.message.trigger.time", 16), SCAN_REPORT_NOTIFY_MESSAGE_TRIGGER_USER("scanner.report.notify.message.trigger.user", 17), - LICENSE_NOT_FOUND("license.not-found",18), - SCAN_REPORT_NOTIFY_MESSAGE_LICENSE("scanner.report.notify.message.license",19), - SCAN_REPORT_NOTIFY_MESSAGE_SENSITIVE("scanner.report.notify.message.sensitive",20), + LICENSE_NOT_FOUND("license.not-found", 18), + SCAN_REPORT_NOTIFY_MESSAGE_LICENSE("scanner.report.notify.message.license", 19), + SCAN_REPORT_NOTIFY_MESSAGE_SENSITIVE("scanner.report.notify.message.sensitive", 20), EXPORT_REPORT_FAIL("export.report.fail", 21), EXPORT_REPORT_STATUS_INIT("export.report.status.init", 22), EXPORT_REPORT_STATUS_RUNNING("export.report.status.running", 23), @@ -59,7 +59,8 @@ enum class ScannerMessageCode( EXPORT_REPORT_STATUS_QUALITY_PASS("export.report.status.quality.pass", 27), EXPORT_REPORT_STATUS_QUALITY_UN_PASS("export.report.status.quality.un.pass", 28), EXPORT_REPORT_STATUS_FAILED("export.report.status.failed", 29), - ANALYST_ARTIFACT_DELETED("analyst.artifact.deleted", 30); + ANALYST_ARTIFACT_DELETED("analyst.artifact.deleted", 30), + ANALYST_TASK_EXCEED_MAX_GLOBAL_TASK_COUNT("analyst.task.global.count.exceed", 31); override fun getBusinessCode() = businessCode override fun getKey() = key diff --git a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/ScanTask.kt b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/ScanTask.kt index 88e92ae4f6..0fe2069dfb 100644 --- a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/ScanTask.kt +++ b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/ScanTask.kt @@ -37,8 +37,10 @@ data class ScanTask( val name: String? = null, @ApiModelProperty("任务id") val taskId: String, - @ApiModelProperty("项目id") + @ApiModelProperty("项目id，只扫描单个项目时有值") val projectId: String?, + @ApiModelProperty("项目id") + val projectIds: Set, @ApiModelProperty("触发者") val createdBy: String, @ApiModelProperty("最后修改时间") @@ -79,4 +81,6 @@ data class ScanTask( val force: Boolean = false, @ApiModelProperty("扫描任务元数据") val metadata: List -) +) { + fun isGlobal() = metadata.any { it.key == TaskMetadata.TASK_METADATA_GLOBAL && it.value == "true" } +} diff --git a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/TaskMetadata.kt b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/TaskMetadata.kt index 9e2bb317c1..0465a0572e 100644 --- a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/TaskMetadata.kt +++ b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/TaskMetadata.kt @@ -66,5 +66,10 @@ data class TaskMetadata( * 指定任务使用哪个分发器分发子任务 */ const val TASK_METADATA_DISPATCHER = "DISPATCHER" + + /** + * 标记任务是否为全局扫描任务 + */ + const val TASK_METADATA_GLOBAL = "GLOBAL" } } diff --git a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/request/GlobalScanRequest.kt b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/request/GlobalScanRequest.kt new file mode 100644 index 0000000000..a8fca88811 --- /dev/null +++ b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/request/GlobalScanRequest.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.analyst.pojo.request + +import com.tencent.bkrepo.analyst.pojo.TaskMetadata +import com.tencent.bkrepo.common.query.model.Rule +import com.tencent.bkrepo.repository.pojo.project.ProjectMetadata +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("全局扫描请求") +data class GlobalScanRequest( + @ApiModelProperty("扫描器名") + val scanner: String, + @ApiModelProperty("扫描文件匹配规则") + val rule: Rule, + @ApiModelProperty("是否强制扫描，为true时无论是否存在扫描结果都会执行扫描") + val force: Boolean = false, + @ApiModelProperty("项目元数据，用于筛选待扫描项目") + val projectMetadata: List = emptyList(), + @ApiModelProperty("任务元数据") + val metadata: List = emptyList() +) diff --git a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/request/ScanTaskQuery.kt b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/request/ScanTaskQuery.kt index 4bb81c6858..3bb90d8c45 100644 --- a/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/request/ScanTaskQuery.kt +++ b/src/backend/analyst/api-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/pojo/request/ScanTaskQuery.kt @@ -33,7 +33,7 @@ import io.swagger.annotations.ApiModelProperty @ApiModel("获取扫描任务") data class ScanTaskQuery( @ApiModelProperty("任务所属项目") - val projectId: String, + val projectId: String? = null, @ApiModelProperty("任务名前缀") val namePrefix: String? = null, @ApiModelProperty("扫描方案id") diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt index 2037c6879d..383bda3669 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt @@ -39,6 +39,7 @@ import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.analyst.model.SubScanTaskDefinition +import com.tencent.bkrepo.common.security.permission.PrincipalType import org.springframework.context.annotation.Primary import org.springframework.stereotype.Component import org.springframework.web.servlet.HandlerMapping @@ -62,6 +63,10 @@ class ScannerPermissionCheckHandler( permissionManager.checkPrincipal(userId, principal.type) } + fun checkPrincipal(userId: String, principal: PrincipalType) { + permissionManager.checkPrincipal(userId, principal) + } + fun checkProjectPermission( projectId: String, action: PermissionAction, diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerProperties.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerProperties.kt index d8688a4c13..1d42e68871 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerProperties.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerProperties.kt @@ -83,6 +83,10 @@ data class ScannerProperties( * 生成的制品临时下载链接超时时间允许下载的次数 */ var tempDownloadUrlPermits: Int? = null, + /** + * 最大全局扫描任务数量 + */ + var maxGlobalTaskCount: Int = 1, ) { companion object { /** diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/controller/user/UserScanController.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/controller/user/UserScanController.kt index 593bc11c1d..990cd32dfb 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/controller/user/UserScanController.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/controller/user/UserScanController.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.analyst.controller.user import com.tencent.bkrepo.analyst.pojo.ScanTask import com.tencent.bkrepo.analyst.pojo.ScanTriggerType +import com.tencent.bkrepo.analyst.pojo.request.GlobalScanRequest import com.tencent.bkrepo.analyst.pojo.request.PipelineScanRequest import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType @@ -49,6 +50,8 @@ import com.tencent.bkrepo.analyst.pojo.response.SubtaskResultOverview import com.tencent.bkrepo.analyst.service.ScanService import com.tencent.bkrepo.analyst.service.ScanTaskService import com.tencent.bkrepo.analyst.utils.ScanPlanConverter +import com.tencent.bkrepo.common.security.permission.Principal +import com.tencent.bkrepo.common.security.permission.PrincipalType import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam @@ -69,6 +72,13 @@ class UserScanController @Autowired constructor( private val scanTaskService: ScanTaskService ) { + @ApiOperation("手动创建全局扫描任务") + @PostMapping("/global") + @Principal(PrincipalType.ADMIN) + fun globalScan(@RequestBody scanRequest: GlobalScanRequest): Response { + return ResponseBuilder.success(scanService.globalScan(scanRequest)) + } + @ApiOperation("手动创建扫描任务") @PostMapping fun scan(@RequestBody scanRequest: ScanRequest): Response { @@ -117,6 +127,16 @@ class UserScanController @Autowired constructor( return ResponseBuilder.success(scanService.stopTask(projectId, taskId)) } + @ApiOperation("中止制品扫描") + @PostMapping("/tasks/{taskId}/stop") + @Principal(PrincipalType.ADMIN) + fun stopTask( + @ApiParam(value = "任务id") + @PathVariable("taskId") taskId: String + ): Response { + return ResponseBuilder.success(scanService.stopTask(null, taskId)) + } + @ApiOperation("获取扫描任务信息") @GetMapping("/tasks/{taskId}") fun task(@PathVariable("taskId") taskId: String): Response { diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dao/ScanTaskDao.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dao/ScanTaskDao.kt index 68e1ab1651..bd12100f89 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dao/ScanTaskDao.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dao/ScanTaskDao.kt @@ -30,6 +30,7 @@ package com.tencent.bkrepo.analyst.dao import com.mongodb.client.result.UpdateResult import com.tencent.bkrepo.analyst.model.TScanTask import com.tencent.bkrepo.analyst.pojo.ScanTaskStatus +import com.tencent.bkrepo.analyst.pojo.TaskMetadata import com.tencent.bkrepo.analyst.pojo.request.ScanTaskQuery import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.api.util.ofTimestamp @@ -41,6 +42,7 @@ import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.elemMatch import org.springframework.data.mongodb.core.query.inValues import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Repository @@ -180,12 +182,10 @@ class ScanTaskDao : ScannerSimpleMongoDao() { return find(Query(Criteria.where(ID).inValues(ids))) } - fun findByProjectIdAndId(projectId: String, id: String): TScanTask? { - return findOne( - Query( - TScanTask::projectId.isEqualTo(projectId).and(ID).isEqualTo(id) - ) - ) + fun findByProjectIdAndId(projectId: String?, id: String): TScanTask? { + val criteria = projectId?.let { TScanTask::projectId.isEqualTo(it).and(ID).isEqualTo(id) } + ?: Criteria().and(ID).isEqualTo(id).and(TScanTask::metadata.name).elemMatch(buildGlobalTaskCriteria()) + return findOne(Query(criteria)) } fun findUnFinished(projectId: String, planId: String): List { @@ -200,7 +200,8 @@ class ScanTaskDao : ScannerSimpleMongoDao() { fun find(scanTaskQuery: ScanTaskQuery, pageLimit: PageLimit): Page { val criteria = Criteria() with(scanTaskQuery) { - criteria.and(TScanTask::projectId.name).isEqualTo(projectId) + projectId?.let { criteria.and(TScanTask::projectId.name).isEqualTo(projectId) } + ?: criteria.and(TScanTask::metadata.name).elemMatch(buildGlobalTaskCriteria()) namePrefix?.let { criteria.and(TScanTask::name.name).regex("^$it") } planId?.let { criteria.and(TScanTask::planId.name).isEqualTo(it) } triggerType?.let { criteria.and(TScanTask::triggerType.name).isEqualTo(it) } @@ -230,6 +231,15 @@ class ScanTaskDao : ScannerSimpleMongoDao() { return count(Query(TScanTask::status.isEqualTo(status.name))) } + fun countGlobalTask(status: List): Long { + val criteria = TScanTask::metadata.elemMatch(buildGlobalTaskCriteria()) + .and(TScanTask::status.name).inValues(status) + return count(Query(criteria)) + } + + private fun buildGlobalTaskCriteria() = + TaskMetadata::key.isEqualTo(TaskMetadata.TASK_METADATA_GLOBAL).and(TaskMetadata::value.name).isEqualTo("true") + private fun buildQuery(taskId: String) = Query(Criteria.where(ID).isEqualTo(taskId)) private fun buildUpdate(lastModifiedDate: LocalDateTime = LocalDateTime.now()): Update = diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dao/SubScanTaskDao.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dao/SubScanTaskDao.kt index 51346cd4a5..663a1f337d 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dao/SubScanTaskDao.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/dao/SubScanTaskDao.kt @@ -179,10 +179,13 @@ class SubScanTaskDao( /** * 获取项目[projectId]扫描中的任务数量 */ - fun scanningCount(projectId: String): Long { + fun scanningCount(projectId: String, includeGlobal: Boolean = false): Long { val criteria = Criteria .where(TSubScanTask::projectId.name).isEqualTo(projectId) .and(TSubScanTask::status.name).inValues(SubScanTaskStatus.RUNNING_STATUS) + if (!includeGlobal) { + criteria.and("${TSubScanTask::metadata.name}.key").ne(TaskMetadata.TASK_METADATA_GLOBAL) + } return count(Query(criteria)) } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/model/TScanTask.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/model/TScanTask.kt index c38d0fb14e..ec6a2b7eb6 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/model/TScanTask.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/model/TScanTask.kt @@ -73,9 +73,13 @@ data class TScanTask( */ val planId: String? = null, /** - * 扫描的项目 + * 扫描的项目，扫描单个项目时有值 */ val projectId: String? = null, + /** + * 扫描的项目列表 + */ + val projectIds: Set = emptySet(), /** * 任务状态 */ diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/ProjectScanConfigurationService.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/ProjectScanConfigurationService.kt index 4afd03cff7..847efe3354 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/ProjectScanConfigurationService.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/ProjectScanConfigurationService.kt @@ -81,5 +81,5 @@ interface ProjectScanConfigurationService { * * @return 项目扫描配置 */ - fun findProjectOrGlobalScanConfiguration(projectId: String): ProjectScanConfiguration? + fun findProjectOrGlobalScanConfiguration(projectId: String? = null): ProjectScanConfiguration? } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/ScanService.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/ScanService.kt index 8af61afd42..0ecb152cf8 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/ScanService.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/ScanService.kt @@ -30,6 +30,7 @@ package com.tencent.bkrepo.analyst.service import com.tencent.bkrepo.analyst.pojo.ScanTask import com.tencent.bkrepo.analyst.pojo.ScanTriggerType import com.tencent.bkrepo.analyst.pojo.SubScanTask +import com.tencent.bkrepo.analyst.pojo.request.GlobalScanRequest import com.tencent.bkrepo.analyst.pojo.request.PipelineScanRequest import com.tencent.bkrepo.analyst.pojo.request.ReportResultRequest import com.tencent.bkrepo.analyst.pojo.request.ScanRequest @@ -38,6 +39,15 @@ import com.tencent.bkrepo.analyst.pojo.request.ScanRequest * 扫描服务 */ interface ScanService { + /** + * 触发全局扫描 + * + * @param request 全局扫描请求 + * + * @return 创建的扫描任务 + */ + fun globalScan(request: GlobalScanRequest): ScanTask + /** * 创建扫描任务，启动扫描 * @@ -90,7 +100,7 @@ interface ScanService { * * @return true 停止成功，false 停止失败 */ - fun stopTask(projectId: String, taskId: String): Boolean + fun stopTask(projectId: String?, taskId: String): Boolean /** * 扫描结果上报 diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ProjectScanConfigurationServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ProjectScanConfigurationServiceImpl.kt index 7a54b2353e..6ec95d978d 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ProjectScanConfigurationServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ProjectScanConfigurationServiceImpl.kt @@ -127,9 +127,10 @@ class ProjectScanConfigurationServiceImpl( ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND) } - override fun findProjectOrGlobalScanConfiguration(projectId: String): ProjectScanConfiguration? { - val configuration = projectScanConfigurationDao.findByProjectId(projectId) - ?: projectScanConfigurationDao.findByProjectId(GLOBAL_PROJECT_ID) + override fun findProjectOrGlobalScanConfiguration(projectId: String?): ProjectScanConfiguration? { + val configuration = projectId?.let { + projectScanConfigurationDao.findByProjectId(it) + } ?: projectScanConfigurationDao.findByProjectId(GLOBAL_PROJECT_ID) return configuration?.let { Converter.convert(it) } } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanServiceImpl.kt index 7e193e58b3..41a04ad444 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanServiceImpl.kt @@ -34,17 +34,21 @@ import com.tencent.bkrepo.analyst.configuration.ScannerProperties.Companion.DEFA import com.tencent.bkrepo.analyst.dao.PlanArtifactLatestSubScanTaskDao import com.tencent.bkrepo.analyst.dao.ScanTaskDao import com.tencent.bkrepo.analyst.dao.SubScanTaskDao +import com.tencent.bkrepo.analyst.message.ScannerMessageCode import com.tencent.bkrepo.analyst.model.TSubScanTask import com.tencent.bkrepo.analyst.pojo.ScanTask import com.tencent.bkrepo.analyst.pojo.ScanTaskStatus +import com.tencent.bkrepo.analyst.pojo.ScanTaskStatus.Companion.unFinishedStatus import com.tencent.bkrepo.analyst.pojo.ScanTriggerType import com.tencent.bkrepo.analyst.pojo.SubScanTask import com.tencent.bkrepo.analyst.pojo.TaskMetadata import com.tencent.bkrepo.analyst.pojo.TaskMetadata.Companion.TASK_METADATA_BUILD_NUMBER +import com.tencent.bkrepo.analyst.pojo.TaskMetadata.Companion.TASK_METADATA_GLOBAL import com.tencent.bkrepo.analyst.pojo.TaskMetadata.Companion.TASK_METADATA_KEY_BID import com.tencent.bkrepo.analyst.pojo.TaskMetadata.Companion.TASK_METADATA_KEY_PID import com.tencent.bkrepo.analyst.pojo.TaskMetadata.Companion.TASK_METADATA_PIPELINE_NAME import com.tencent.bkrepo.analyst.pojo.TaskMetadata.Companion.TASK_METADATA_PLUGIN_NAME +import com.tencent.bkrepo.analyst.pojo.request.GlobalScanRequest import com.tencent.bkrepo.analyst.pojo.request.PipelineScanRequest import com.tencent.bkrepo.analyst.pojo.request.ReportResultRequest import com.tencent.bkrepo.analyst.pojo.request.ScanRequest @@ -61,15 +65,21 @@ import com.tencent.bkrepo.analyst.statemachine.task.ScanTaskEvent import com.tencent.bkrepo.analyst.statemachine.task.context.CreateTaskContext import com.tencent.bkrepo.analyst.statemachine.task.context.ResetTaskContext import com.tencent.bkrepo.analyst.statemachine.task.context.StopTaskContext +import com.tencent.bkrepo.analyst.utils.RuleConverter +import com.tencent.bkrepo.analyst.utils.RuleUtil import com.tencent.bkrepo.analyst.utils.SubtaskConverter import com.tencent.bkrepo.common.analysis.pojo.scanner.ScanExecutorResult import com.tencent.bkrepo.common.analysis.pojo.scanner.SubScanTaskStatus import com.tencent.bkrepo.common.analysis.pojo.scanner.SubScanTaskStatus.EXECUTING import com.tencent.bkrepo.common.analysis.pojo.scanner.SubScanTaskStatus.PULLED +import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.lock.service.LockOperation import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.repository.api.ProjectClient +import com.tencent.bkrepo.repository.pojo.project.ProjectRangeQueryRequest import com.tencent.bkrepo.statemachine.Event import com.tencent.bkrepo.statemachine.StateMachine import org.slf4j.LoggerFactory @@ -97,8 +107,39 @@ class ScanServiceImpl @Autowired constructor( private val redisTemplate: RedisTemplate, private val reportExporter: ReportExporter, private val scannerProperties: ScannerProperties, + private val projectClient: ProjectClient, + private val lockOperation: LockOperation, ) : ScanService { + override fun globalScan(request: GlobalScanRequest): ScanTask { + return with(request) { + lockOperation.doWithLock(GLOBAL_SCAN_LOCK) { + if (scanTaskDao.countGlobalTask(unFinishedStatus) >= scannerProperties.maxGlobalTaskCount) { + throw ErrorCodeException(ScannerMessageCode.ANALYST_TASK_EXCEED_MAX_GLOBAL_TASK_COUNT) + } + + // projectId需要在第一层 + var projectIds = RuleUtil.getProjectIds(rule) + if (projectMetadata.isNotEmpty()) { + val metadataProjectIds = projectClient.rangeQuery( + ProjectRangeQueryRequest(emptyList(), projectMetadata = projectMetadata) + ).data?.records?.map { it?.name!! } ?: emptyList() + projectIds = projectIds + metadataProjectIds + } + val scanRequest = ScanRequest( + scanner = scanner, + rule = RuleConverter.convert(rule, null, projectIds), + force = force, + metadata = metadata + listOf(TaskMetadata(TASK_METADATA_GLOBAL, "true")) + ) + val userId = SecurityUtils.getUserId() + val task = scan(scanRequest, ScanTriggerType.MANUAL, userId) + logger.info("User[$userId] triggered global scan task[${task.taskId}]") + task + } + } + } + override fun scan(scanRequest: ScanRequest, triggerType: ScanTriggerType, userId: String): ScanTask { val context = CreateTaskContext(scanRequest = scanRequest, triggerType = triggerType, userId = userId) val event = Event(ScanTaskEvent.CREATE.name, context) @@ -156,7 +197,7 @@ class ScanServiceImpl @Autowired constructor( } @Transactional(rollbackFor = [Throwable::class]) - override fun stopTask(projectId: String, taskId: String): Boolean { + override fun stopTask(projectId: String?, taskId: String): Boolean { val task = scanTaskDao.findByProjectIdAndId(projectId, taskId) ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND) @@ -265,7 +306,7 @@ class ScanServiceImpl @Autowired constructor( if (task.executedTimes >= DEFAULT_MAX_EXECUTE_TIMES || System.currentTimeMillis() >= expiredTimestamp) { logger.info( "subTask[${task.id}] of parentTask[${task.parentScanTaskId}] " + - "exceed max execute times or timeout[${task.lastModifiedDate}]" + "exceed max execute times or timeout[${task.lastModifiedDate}]" ) val targetState = if (task.status == EXECUTING.name || task.status == PULLED.name) { SubScanTaskStatus.TIMEOUT.name @@ -308,6 +349,8 @@ class ScanServiceImpl @Autowired constructor( companion object { private val logger = LoggerFactory.getLogger(ScanServiceImpl::class.java) + private const val GLOBAL_SCAN_LOCK = "lock:global:scan" + /** * 最大允许重复执行次数 */ diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt index 53a990ec73..a164ca3d52 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/service/impl/ScanTaskServiceImpl.kt @@ -121,7 +121,11 @@ class ScanTaskServiceImpl( } override fun tasks(scanTaskQuery: ScanTaskQuery, pageLimit: PageLimit): Page { - permissionCheckHandler.checkProjectPermission(scanTaskQuery.projectId, PermissionAction.MANAGE) + if (scanTaskQuery.projectId == null) { + permissionCheckHandler.checkPrincipal(SecurityUtils.getUserId(), PrincipalType.ADMIN) + } else { + permissionCheckHandler.checkProjectPermission(scanTaskQuery.projectId!!, PermissionAction.MANAGE) + } val taskPage = scanTaskDao.find(scanTaskQuery, pageLimit) val records = taskPage.records.map { Converter.convert(it) } return Page(pageLimit.pageNumber, pageLimit.pageSize, taskPage.totalRecords, records) diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/NodeIterator.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/NodeIterator.kt index 5709f142c9..f6686ce562 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/NodeIterator.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/iterator/NodeIterator.kt @@ -34,6 +34,7 @@ import com.tencent.bkrepo.common.query.model.Rule import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.analyst.utils.Request +import org.slf4j.LoggerFactory /** * 文件迭代器 @@ -87,7 +88,12 @@ class NodeIterator( val projectIdRule = modifyRule(projectId, rule) // 获取下一页需要扫描的文件 - return Request.requestNodes(nodeClient, projectIdRule, page, pageSize) + return try { + Request.requestNodes(nodeClient, projectIdRule, page, pageSize) + } catch (e: Exception) { + logger.error("iterate node failed, projectId[$projectId], page[$page], pageSize[$pageSize], rule[$rule]", e) + emptyList() + } } /** @@ -148,4 +154,8 @@ class NodeIterator( override var pageSize: Int = DEFAULT_PAGE_SIZE, override var index: Int = INITIAL_INDEX ) : PageIteratePosition(page = page, pageSize = pageSize, index = index) + + companion object { + private val logger = LoggerFactory.getLogger(NodeIterator::class.java) + } } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/task/action/PendingAction.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/task/action/PendingAction.kt index 1fadfd5dce..d9908293fc 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/task/action/PendingAction.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/task/action/PendingAction.kt @@ -60,6 +60,7 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.query.model.Rule +import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.service.util.LocaleMessageUtils.getLocalizedMessage import com.tencent.bkrepo.statemachine.Event import com.tencent.bkrepo.statemachine.StateMachine @@ -117,19 +118,26 @@ class PendingAction( } val plan = planId?.let { scanPlanDao.get(it) } - val projectId = projectId(rule, plan) + val projectIds = parseProjectIds(rule, plan) + val projectId = if (projectIds.size == 1) { + projectIds.first() + } else { + null + } val repoNames = RuleUtil.getRepoNames(rule) val scanner = scannerService.get(scanner ?: plan!!.scanner) val metadata = customMetadata(metadata, projectId, scanner) // 校验权限 - if (repoNames.isEmpty()) { + if (projectId == null) { + permissionCheckHandler.checkPrincipal(userId, PrincipalType.ADMIN) + } else if (repoNames.isEmpty()) { permissionCheckHandler.checkProjectPermission(projectId, PermissionAction.MANAGE, userId) } else { permissionCheckHandler.checkReposPermission(projectId, repoNames, PermissionAction.READ, userId) } - val rule = RuleConverter.convert(rule, plan?.type, projectId) + val rule = RuleConverter.convert(rule, plan?.type, projectIds) val now = LocalDateTime.now() val scanTask = scanTaskDao.save( TScanTask( @@ -142,6 +150,7 @@ class PendingAction( triggerType = triggerType.name, planId = plan?.id, projectId = projectId, + projectIds = projectIds.toSet(), status = ScanTaskStatus.PENDING.name, total = 0L, scanning = 0L, @@ -163,7 +172,7 @@ class PendingAction( } } - private fun customMetadata(metadata: List, projectId: String, scanner: Scanner): List { + private fun customMetadata(metadata: List, projectId: String?, scanner: Scanner): List { val projectScanConfiguration = projectScanConfigurationService.findProjectOrGlobalScanConfiguration(projectId) val customMetadata = metadata.filter { it.key != TASK_METADATA_DISPATCHER } @@ -188,26 +197,27 @@ class PendingAction( val pluginName = metadataMap[TaskMetadata.TASK_METADATA_PLUGIN_NAME]?.value ?: "" "$pipelineName-$buildNo-$pluginName" } + else -> getLocalizedMessage(ScannerMessageCode.SCAN_TASK_NAME_BATCH_SCAN) } } - private fun projectId(rule: Rule?, plan: TScanPlan?): String { + private fun parseProjectIds(rule: Rule?, plan: TScanPlan?): List { // 尝试从rule取projectId，不存在时从plan中取projectId val projectIds = RuleUtil.getProjectIds(rule) - return if (projectIds.size == 1) { - projectIds.first() - } else if (projectIds.isEmpty() && plan != null) { - plan.projectId + return if (projectIds.isNotEmpty()) { + projectIds + } else if (plan != null) { + listOf(plan.projectId) } else { - throw ErrorCodeException(CommonMessageCode.PARAMETER_INVALID) + throw ErrorCodeException(CommonMessageCode.PARAMETER_MISSING, "projectId rule") } } override fun support(from: String, to: String, event: String): Boolean { return from == ScanTaskStatus.PENDING.name - && to == ScanTaskStatus.PENDING.name - && event == ScanTaskEvent.CREATE.name + && to == ScanTaskStatus.PENDING.name + && event == ScanTaskEvent.CREATE.name } companion object { diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/task/action/SubmittingAction.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/task/action/SubmittingAction.kt index e8c2a2130d..28919df8a1 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/task/action/SubmittingAction.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/statemachine/task/action/SubmittingAction.kt @@ -48,6 +48,7 @@ import com.tencent.bkrepo.analyst.statemachine.iterator.IteratorManager import com.tencent.bkrepo.analyst.statemachine.subtask.SubtaskEvent import com.tencent.bkrepo.analyst.statemachine.subtask.context.CreateSubtaskContext import com.tencent.bkrepo.analyst.statemachine.task.ScanTaskEvent +import com.tencent.bkrepo.analyst.statemachine.task.context.StopTaskContext import com.tencent.bkrepo.analyst.statemachine.task.context.SubmitTaskContext import com.tencent.bkrepo.common.analysis.pojo.scanner.SubScanTaskStatus.NEVER_SCANNED import com.tencent.bkrepo.common.lock.service.LockOperation @@ -121,8 +122,11 @@ class SubmittingAction( logger.info("task[${e.taskId}] has been stopped") return TransitResult(ScanTaskStatus.STOPPING.name) } catch (e: Exception) { - logger.error("submit task failed, taskId[${scanTask.taskId}]", e) - throw e + logger.error("submit task[${scanTask.taskId}] failed, try to stop task", e) + return taskStateMachine.sendEvent( + SCANNING_SUBMITTING.name, + Event(ScanTaskEvent.STOP.name, StopTaskContext(scanTaskDao.findById(scanTask.taskId)!!)) + ) } logger.info("submit $submittedSubTaskCount sub tasks, $reuseResultTaskCount sub tasks reuse result") @@ -174,7 +178,7 @@ class SubmittingAction( reuseResultTaskCount++ } else { // 变更子任务状态为CREATED或BLOCKED - val event = event(scanningCount, projectScanConfiguration) + val event = event(scanTask, scanningCount, projectScanConfiguration) subtaskStateMachine.sendEvent(NEVER_SCANNED.name, Event(event.name, context)) // 统计子任务数量 submittedSubTaskCount++ @@ -204,9 +208,14 @@ class SubmittingAction( * */ private fun event( + scanTask: ScanTask, scanningCount: Long, projectConfiguration: ProjectScanConfiguration? ): SubtaskEvent { + if (scanTask.isGlobal()) { + // 全局任务不阻塞 + return SubtaskEvent.CREATE + } val limitSubScanTaskCount = projectConfiguration?.subScanTaskCountLimit ?: scannerProperties.defaultProjectSubScanTaskCountLimit if (scanningCount >= limitSubScanTaskCount.toLong()) { diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/Converter.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/Converter.kt index e88cd22aa1..178da58a8b 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/Converter.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/Converter.kt @@ -59,6 +59,7 @@ object Converter { name = scanTaskName(triggerType, scanTask.name), taskId = id!!, projectId = projectId, + projectIds = projectIds, createdBy = createdBy, lastModifiedDateTime = lastModifiedDate.format(DateTimeFormatter.ISO_DATE_TIME), triggerDateTime = createdDate.format(DateTimeFormatter.ISO_DATE_TIME), diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/RuleConverter.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/RuleConverter.kt index b5950d4e47..9126881603 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/RuleConverter.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/RuleConverter.kt @@ -39,26 +39,26 @@ import com.tencent.bkrepo.analyst.pojo.rule.RuleArtifact object RuleConverter { - fun convert(sourceRule: Rule?, planType: String?, projectId: String): Rule { + fun convert(sourceRule: Rule?, planType: String?, projectIds: List): Rule { // 兼容许可证扫描的planType:MAVEN_LICENSE - val targetRule = createProjectIdAndRepoRule(projectId, emptyList(), planType) + val targetRule = createProjectIdAndRepoRule(projectIds, emptyList(), planType) // 将sourceRule中除projectId相关外的rule都合并到targetRule中 sourceRule?.let { mergeInto(it, targetRule, listOf(NodeInfo::projectId.name)) } return targetRule } fun convert(projectId: String, repoNames: List, repoType: String? = null): Rule { - return createProjectIdAndRepoRule(projectId, repoNames, repoType) + return createProjectIdAndRepoRule(listOf(projectId), repoNames, repoType) } fun convert(projectId: String, repoName: String, fullPath: String): Rule { - val rule = createProjectIdAndRepoRule(projectId, listOf(repoName)) + val rule = createProjectIdAndRepoRule(listOf(projectId), listOf(repoName)) rule.rules.add(Rule.QueryRule(NodeDetail::fullPath.name, fullPath, OperationType.EQ)) return rule } fun convert(projectId: String, repoName: String, packageKey: String, version: String): Rule { - val rule = createProjectIdAndRepoRule(projectId, listOf(repoName)) + val rule = createProjectIdAndRepoRule(listOf(projectId), listOf(repoName)) rule.rules.add(Rule.QueryRule(PackageSummary::key.name, packageKey, OperationType.EQ)) rule.rules.add(Rule.QueryRule(RuleArtifact::version.name, version, OperationType.EQ)) return rule @@ -68,13 +68,15 @@ object RuleConverter { * 添加projectId和repoName规则 */ private fun createProjectIdAndRepoRule( - projectId: String, + projectIds: List, repoNames: List, repoType: String? = null ): NestedRule { - val rules = mutableListOf( - Rule.QueryRule(NodeDetail::projectId.name, projectId, OperationType.EQ) - ) + val rules = if (projectIds.size == 1) { + mutableListOf(Rule.QueryRule(NodeDetail::projectId.name, projectIds.first(), OperationType.EQ)) + } else { + mutableListOf(Rule.QueryRule(NodeDetail::projectId.name, projectIds, OperationType.IN)) + } if (repoType != null && repoType != RepositoryType.GENERIC.name) { rules.add(Rule.QueryRule(PackageSummary::type.name, repoType, OperationType.EQ)) diff --git a/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_en.properties b/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_en.properties index 5849d09068..fe795599c1 100644 --- a/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_en.properties +++ b/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_en.properties @@ -53,3 +53,4 @@ export.report.status.quality.pass=quality rules passed export.report.status.quality.un.pass=quality rule failed export.report.status.failed=scan anomalies analyst.artifact.deleted=Artifact [{0}] was deleted +analyst.task.global.count.exceed=Global task count has reached the limit, new global task cannot be created. diff --git a/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_zh_CN.properties index fb922c710f..e0e02f656e 100644 --- a/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_zh_CN.properties @@ -53,3 +53,4 @@ export.report.status.quality.pass=质量规则通过 export.report.status.quality.un.pass=质量规则未通过 export.report.status.failed=扫描异常 analyst.artifact.deleted=制品 [{0}] 已被删除 +analyst.task.global.count.exceed=当前执行中的全局任务数达到限制，无法创建新的全局任务 diff --git a/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_zh_TW.properties index 10cec731a8..685a0780b9 100644 --- a/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/analyst/biz-analyst/src/main/resources/i18n/messages_zh_TW.properties @@ -53,3 +53,4 @@ export.report.status.quality.pass=質量規則通過 export.report.status.quality.un.pass=質量規則未通過 export.report.status.failed=掃描異常 analyst.artifact.deleted=製品 [{0}] 已刪除 +analyst.task.global.count.exceed=目前執行中的全域任務數達到限制，無法建立新的全域任務 diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/project/ProjectRangeQueryRequest.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/project/ProjectRangeQueryRequest.kt index d98561ea84..eaf9419f33 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/project/ProjectRangeQueryRequest.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/project/ProjectRangeQueryRequest.kt @@ -41,5 +41,7 @@ data class ProjectRangeQueryRequest( @ApiModelProperty("分页偏移量", required = false) val offset: Long = 0L, @ApiModelProperty("分页大小", required = false) - val limit: Int = 20 + val limit: Int = 20, + @ApiModelProperty("项目元数据", required = false) + val projectMetadata: List = emptyList(), ) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/ProjectServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/ProjectServiceImpl.kt index d84f19086b..a5dbb6f4a8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/ProjectServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/ProjectServiceImpl.kt @@ -48,6 +48,7 @@ import com.tencent.bkrepo.repository.model.TProjectMetrics import com.tencent.bkrepo.repository.pojo.project.ProjectCreateRequest import com.tencent.bkrepo.repository.pojo.project.ProjectInfo import com.tencent.bkrepo.repository.pojo.project.ProjectListOption +import com.tencent.bkrepo.repository.pojo.project.ProjectMetadata import com.tencent.bkrepo.repository.pojo.project.ProjectMetricsInfo import com.tencent.bkrepo.repository.pojo.project.ProjectRangeQueryRequest import com.tencent.bkrepo.repository.pojo.project.ProjectSearchOption @@ -64,7 +65,9 @@ import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.elemMatch import org.springframework.data.mongodb.core.query.inValues +import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.regex import org.springframework.data.mongodb.core.query.where import org.springframework.stereotype.Service @@ -159,18 +162,23 @@ class ProjectServiceImpl( override fun rangeQuery(request: ProjectRangeQueryRequest): Page { val limit = request.limit val skip = request.offset - return if (request.projectIds.isEmpty()) { - val query = Query() - val totalCount = projectDao.count(query) - val records = projectDao.find(query.skip(skip).limit(limit)).map { convert(it) } - Page(0, limit, totalCount, records) + var criteria = if (request.projectIds.isEmpty()) { + Criteria() } else { - val criteria = TProject::name.inValues(request.projectIds) - val query = Query(criteria) - val totalCount = projectDao.count(query) - val records = projectDao.find(query.limit(limit).skip(skip)).map { convert(it) } - Page(0, limit, totalCount, records) + TProject::name.inValues(request.projectIds) } + if (request.projectMetadata.isNotEmpty()) { + val metadataCriteria = request.projectMetadata.map { + TProject::metadata.elemMatch( + ProjectMetadata::key.isEqualTo(it.key).and(ProjectMetadata::value.name).isEqualTo(it.value) + ) + } + criteria = Criteria().andOperator(metadataCriteria + criteria) + } + val query = Query(criteria) + val totalCount = projectDao.count(query) + val records = projectDao.find(query.limit(limit).skip(skip)).map { convert(it) } + return Page(0, limit, totalCount, records) } override fun checkExist(name: String): Boolean { diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ProjectServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ProjectServiceTest.kt index 97123e6551..65d4bebfe8 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ProjectServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ProjectServiceTest.kt @@ -39,6 +39,7 @@ import com.tencent.bkrepo.repository.UT_USER import com.tencent.bkrepo.repository.dao.ProjectDao import com.tencent.bkrepo.repository.pojo.project.ProjectCreateRequest import com.tencent.bkrepo.repository.pojo.project.ProjectMetadata +import com.tencent.bkrepo.repository.pojo.project.ProjectRangeQueryRequest import com.tencent.bkrepo.repository.pojo.project.ProjectUpdateRequest import com.tencent.bkrepo.repository.service.repo.ProjectService import org.junit.jupiter.api.Assertions @@ -109,7 +110,7 @@ class ProjectServiceTest @Autowired constructor( Assertions.assertEquals(newDisplayName, newProject!!.displayName) Assertions.assertEquals(newDes, newProject.description) Assertions.assertEquals(newMetadata.size, newProject.metadata.size) - for(i in newMetadata.indices) { + for (i in newMetadata.indices) { Assertions.assertEquals(newMetadata[i], newProject.metadata[i]) } } @@ -170,6 +171,72 @@ class ProjectServiceTest @Autowired constructor( projectService.createProject(request) } + @Test + @DisplayName("测试查询项目") + fun `test range query projects`() { + var request = ProjectCreateRequest( + "p1", "p1", "p1", true, UT_USER, + listOf(ProjectMetadata("bg", "bg1"), ProjectMetadata("department", "dp1")) + ) + projectService.createProject(request) + + request = ProjectCreateRequest( + "p2", "p2", "p2", true, UT_USER, + listOf(ProjectMetadata("bg", "bg2"), ProjectMetadata("department", "dp2")) + ) + projectService.createProject(request) + + // 获取所有项目 + var records = projectService.rangeQuery(ProjectRangeQueryRequest(emptyList())).records + Assertions.assertEquals(2, records.size) + + // 通过projectId查询 + records = projectService.rangeQuery(ProjectRangeQueryRequest(listOf("p2"))).records + Assertions.assertEquals(1, records.size) + Assertions.assertEquals("p2", records.first()!!.name) + + // 通过metadata查询 + var query = ProjectRangeQueryRequest( + projectIds = emptyList(), + projectMetadata = listOf(ProjectMetadata("bg", "bg2")) + ) + records = projectService.rangeQuery(query).records + Assertions.assertEquals(1, records.size) + Assertions.assertEquals("p2", records.first()!!.name) + + query = ProjectRangeQueryRequest( + projectIds = emptyList(), + projectMetadata = listOf(ProjectMetadata("bg", "bg1"), ProjectMetadata("department", "dp1")) + ) + records = projectService.rangeQuery(query).records + Assertions.assertEquals(1, records.size) + Assertions.assertEquals("p1", records.first()!!.name) + + query = ProjectRangeQueryRequest( + projectIds = emptyList(), + projectMetadata = listOf(ProjectMetadata("bg", "bg2"), ProjectMetadata("department", "dp1")) + ) + records = projectService.rangeQuery(query).records + Assertions.assertEquals(0, records.size) + + + // projectId + metadata查询 + query = ProjectRangeQueryRequest( + projectIds = listOf("p2"), + projectMetadata = listOf(ProjectMetadata("bg", "bg2")) + ) + records = projectService.rangeQuery(query).records + Assertions.assertEquals(1, records.size) + Assertions.assertEquals("p2", records.first()!!.name) + + query = ProjectRangeQueryRequest( + projectIds = listOf("p1"), + projectMetadata = listOf(ProjectMetadata("bg", "bg2")) + ) + records = projectService.rangeQuery(query).records + Assertions.assertEquals(0, records.size) + } + private fun removeAllProject() { projectDao.remove(Query()) }