feat: Node提取到common-metadata TencentBlueKing#2619

* feat: Node提取到common-metadata TencentBlueKing#2619 * feat: Node提取到common-metadata TencentBlueKing#2619 * feat: Node提取到common-metadata TencentBlueKing#2619 * feat: Node提取到common-metadata TencentBlueKing#2619 * feat: Node提取到common-metadata TencentBlueKing#2619
lannoy0523 · Oct 25, 2024 · 8173632 · 8173632
1 parent a06d107
commit 8173632
Show file tree

Hide file tree

Showing 263 changed files with 3,388 additions and 2,153 deletions.
diff --git a/...sis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt b/...sis-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/component/FileLoader.kt
@@ -39,11 +39,13 @@ import com.tencent.bkrepo.common.api.exception.SystemErrorException
 import com.tencent.bkrepo.common.api.message.CommonMessageCode
 import com.tencent.bkrepo.common.api.util.readJsonString
 import com.tencent.bkrepo.common.api.util.toJsonString
+import com.tencent.bkrepo.common.artifact.api.ArtifactInfo
 import com.tencent.bkrepo.common.artifact.manager.StorageManager
 import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
+import com.tencent.bkrepo.common.metadata.service.node.NodeSearchService
+import com.tencent.bkrepo.common.metadata.service.node.NodeService
+import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
-import com.tencent.bkrepo.repository.api.NodeClient
-import com.tencent.bkrepo.repository.api.StorageCredentialsClient
 import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder
 import org.apache.commons.codec.binary.Hex
@@ -64,9 +66,10 @@ import java.security.MessageDigest
 @Component
 class FileLoader(
     private val executorProperties: ScannerExecutorProperties,
-    private val nodeClient: NodeClient,
+    private val nodeService: NodeService,
+    private val nodeSearchService: NodeSearchService,
     private val storageManager: StorageManager,
-    private val storageCredentialsClient: StorageCredentialsClient,
+    private val storageCredentialService: StorageCredentialService,
 ) {
     /**
      * 加载[subtask]要扫描的制品
@@ -82,8 +85,8 @@ class FileLoader(
             }
 
             // 获取存储凭据
-            val storageCredentials = credentialsKey?.let { storageCredentialsClient.findByKey(it).data!! }
-            val node = nodeClient.getNodeDetail(projectId, repoName, fullPath).data
+            val storageCredentials = credentialsKey?.let { storageCredentialService.findByKey(it)!! }
+            val node = nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))
             // 获取文件
             val file = File(tempDir, fileName(taskId, fileName(), repoType))
             val fos = DigestOutputStream(file.outputStream(), MessageDigest.getInstance("SHA-256"))
@@ -173,7 +176,7 @@ class FileLoader(
     }
 
     private fun getNode(projectId: String, repoName: String, sha256: String): NodeDetail? {
-        val nodes = nodeClient.queryWithoutCount(
+        val nodes = nodeSearchService.searchWithoutCount(
             NodeQueryBuilder()
                 .projectId(projectId)
                 .repoName(repoName)
@@ -182,11 +185,11 @@ class FileLoader(
                 .page(1, 1)
                 .build()
         )
-        if (nodes.isNotOk() || nodes.data!!.records.isEmpty()) {
+        if (nodes.records.isEmpty()) {
             throw SystemErrorException(CommonMessageCode.RESOURCE_NOT_FOUND, sha256)
         }
-        val fullPath = nodes.data!!.records[0][NodeDetail::fullPath.name].toString()
-        return nodeClient.getNodeDetail(projectId, repoName, fullPath).data
+        val fullPath = nodes.records[0][NodeDetail::fullPath.name].toString()
+        return nodeService.getNodeDetail(ArtifactInfo(projectId, repoName, fullPath))
     }
 
     /**

diff --git a/...-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt b/...-executor/src/main/kotlin/com/tencent/bkrepo/analysis/executor/trivy/TrivyScanExecutor.kt
@@ -50,12 +50,13 @@ import com.tencent.bkrepo.common.api.exception.NotFoundException
 import com.tencent.bkrepo.common.api.exception.SystemErrorException
 import com.tencent.bkrepo.common.api.message.CommonMessageCode.RESOURCE_NOT_FOUND
 import com.tencent.bkrepo.common.api.message.CommonMessageCode.SYSTEM_ERROR
+import com.tencent.bkrepo.common.artifact.api.ArtifactInfo
 import com.tencent.bkrepo.common.artifact.hash.md5
 import com.tencent.bkrepo.common.artifact.manager.StorageManager
 import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
+import com.tencent.bkrepo.common.metadata.service.node.NodeService
+import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService
 import com.tencent.bkrepo.common.query.model.Sort
-import com.tencent.bkrepo.repository.api.NodeClient
-import com.tencent.bkrepo.repository.api.RepositoryClient
 import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.node.NodeInfo
 import com.tencent.bkrepo.repository.pojo.node.NodeListOption
@@ -71,9 +72,9 @@ import java.io.FileOutputStream
 class TrivyScanExecutor @Autowired constructor(
     dockerClient: DockerClient,
     private val scannerExecutorProperties: ScannerExecutorProperties,
-    private val repositoryClient: RepositoryClient,
+    private val repositoryService: RepositoryService,
     private val storageManager: StorageManager,
-    private val nodeClient: NodeClient
+    private val nodeService: NodeService
 ) : CommonScanExecutor() {
 
     private val dockerScanHelper = DockerScanHelper(scannerExecutorProperties, dockerClient)
@@ -169,19 +170,11 @@ class TrivyScanExecutor @Autowired constructor(
         val scanner = task.scanner
         require(scanner is TrivyScanner)
         // 获取trivy默认仓库信息
-        val repoRes = repositoryClient.getRepoDetail(scanner.vulDbConfig.projectId, scanner.vulDbConfig.repo)
-        if (repoRes.isNotOk()) {
-            logger.error(
-                "Get repo info failed: code[${repoRes.code}], message[${repoRes.message}]," +
-                        " projectId[${scanner.vulDbConfig.projectId}], repoName[${scanner.vulDbConfig.repo}]"
-            )
-            throw SystemErrorException(SYSTEM_ERROR, repoRes.message ?: "")
-        }
-        val repositoryDetail = repoRes.data
+        val repositoryDetail = repositoryService.getRepoDetail(scanner.vulDbConfig.projectId, scanner.vulDbConfig.repo)
             ?: throw NotFoundException(RESOURCE_NOT_FOUND, scanner.vulDbConfig.repo)
 
         return storageManager.loadFullArtifactInputStream(NodeDetail(dbNode), repositoryDetail.storageCredentials)
-            ?: throw SystemErrorException(SYSTEM_ERROR, "load trivy.db file failed: res: ${repoRes.message}")
+            ?: throw SystemErrorException(SYSTEM_ERROR, "load trivy.db file failed")
     }
 
     private fun getNewestNode(projectId: String, repo: String): NodeInfo {
@@ -193,16 +186,9 @@ class TrivyScanExecutor @Autowired constructor(
             sortProperty = listOf("lastModifiedDate", "createdDate"),
             direction = listOf(Sort.Direction.DESC.name, Sort.Direction.DESC.name)
         )
-        val nodeRes = nodeClient.listNodePage(projectId, repo, "/trivy", option)
-        if (nodeRes.isNotOk()) {
-            logger.error(
-                "Get node info failed: code[${nodeRes.code}], message[${nodeRes.message}]," +
-                        " projectId[$projectId], repoName[$repo]"
-            )
-            throw SystemErrorException(SYSTEM_ERROR, nodeRes.message ?: "")
-        }
+        val nodeRes = nodeService.listNodePage(ArtifactInfo(projectId, repo, "/trivy"), option)
         // 获取最新的trivy.db
-        val newestDB = nodeRes.data!!.records.firstOrNull()
+        val newestDB = nodeRes.records.firstOrNull()
         if (newestDB == null) {
             logger.error("Get trivy.db file failed")
             throw SystemErrorException(SYSTEM_ERROR, "Get trivy.db file failed")

diff --git a/...analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/CacheableRepositoryClient.kt b/...analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/CacheableRepositoryClient.kt
@@ -31,16 +31,15 @@ import com.google.common.cache.CacheBuilder
 import com.google.common.cache.CacheLoader
 import com.google.common.cache.LoadingCache
 import com.tencent.bkrepo.common.api.exception.NotFoundException
-import com.tencent.bkrepo.common.api.exception.SystemErrorException
 import com.tencent.bkrepo.common.api.message.CommonMessageCode
-import com.tencent.bkrepo.repository.api.RepositoryClient
+import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService
 import com.tencent.bkrepo.repository.pojo.repo.RepositoryInfo
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Component
 import java.util.concurrent.TimeUnit
 
 @Component
-class CacheableRepositoryClient(private val repositoryClient: RepositoryClient) {
+class CacheableRepositoryClient(private val repositoryService: RepositoryService) {
     private val repoInfoCache: LoadingCache<String, RepositoryInfo> = CacheBuilder.newBuilder()
         .maximumSize(DEFAULT_REPO_INFO_CACHE_SIZE)
         .expireAfterWrite(DEFAULT_REPO_INFO_CACHE_DURATION_MINUTES, TimeUnit.MINUTES)
@@ -52,15 +51,8 @@ class CacheableRepositoryClient(private val repositoryClient: RepositoryClient)
 
     private fun loadRepoInfo(key: String): RepositoryInfo {
         val (projectId, repoName) = fromKey(key)
-        val repoRes = repositoryClient.getRepoInfo(projectId, repoName)
-        if (repoRes.isNotOk()) {
-            logger.error(
-                "Get repo info failed: code[${repoRes.code}], message[${repoRes.message}]," +
-                    " projectId[$projectId], repoName[$repoName]"
-            )
-            throw SystemErrorException(CommonMessageCode.SYSTEM_ERROR, repoRes.message ?: "")
-        }
-        return repoRes.data ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, key)
+        val repo = repositoryService.getRepoInfo(projectId, repoName)
+        return repo ?: throw NotFoundException(CommonMessageCode.RESOURCE_NOT_FOUND, key)
     }
 
     private fun fromKey(key: String): Pair<String, String> {

diff --git a/...yst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt b/...yst/src/main/kotlin/com/tencent/bkrepo/analyst/component/ScannerPermissionCheckHandler.kt
@@ -39,6 +39,7 @@ import com.tencent.bkrepo.common.security.permission.Principal
 import com.tencent.bkrepo.common.security.util.SecurityUtils
 import com.tencent.bkrepo.common.service.util.HttpContextHolder
 import com.tencent.bkrepo.analyst.model.SubScanTaskDefinition
+import com.tencent.bkrepo.common.artifact.pojo.RepositoryId
 import com.tencent.bkrepo.common.security.permission.PrincipalType
 import org.springframework.context.annotation.Primary
 import org.springframework.stereotype.Component
@@ -153,5 +154,5 @@ class ScannerPermissionCheckHandler(
     }
 
     private fun repoDetail(projectId: String, repoName: String) =
-        ArtifactContextHolder.getRepoDetail(ArtifactContextHolder.RepositoryId(projectId, repoName))
+        ArtifactContextHolder.getRepoDetail(RepositoryId(projectId, repoName))
 }
diff --git a/.../main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/ArrowheadConverter.kt b/.../main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/ArrowheadConverter.kt
@@ -48,6 +48,7 @@ import com.tencent.bkrepo.common.api.pojo.Page
 import com.tencent.bkrepo.common.mongo.dao.util.Pages
 import com.tencent.bkrepo.common.query.model.PageLimit
 import org.springframework.stereotype.Component
+import java.util.Locale
 
 @Component("${ArrowheadScanner.TYPE}Converter")
 class ArrowheadConverter(private val licenseService: SpdxLicenseService) : ScannerConverter {
@@ -57,10 +58,10 @@ class ArrowheadConverter(private val licenseService: SpdxLicenseService) : Scann
         result as Page<ApplicationItem>
         // 查询数据库中存放的applicationItem时已经过滤了只存在license的项，license一定存在
         val licenseIds = result.records.map { it.license!!.name }.distinct()
-        val licenses = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.toLowerCase() }
+        val licenses = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.lowercase(Locale.getDefault()) }
 
         val reports = result.records.map {
-            val detail = licenses[it.license!!.name.toLowerCase()]
+            val detail = licenses[it.license!!.name.lowercase(Locale.getDefault())]
             FileLicensesResultDetail(
                 licenseId = it.license!!.name,
                 fullName = detail?.name ?: "",
@@ -144,9 +145,10 @@ class ArrowheadConverter(private val licenseService: SpdxLicenseService) : Scann
         overview[LicenseOverviewKey.overviewKeyOf(LicenseOverviewKey.TOTAL)] = licenses.size.toLong()
 
         // 获取许可证详情
-        val licenseInfo = licenseService.listLicenseByIds(licenseIds.toList()).mapKeys { it.key.toLowerCase() }
+        val licenseInfo =
+            licenseService.listLicenseByIds(licenseIds.toList()).mapKeys { it.key.lowercase(Locale.getDefault()) }
         for (license in licenses) {
-            val detail = licenseInfo[license.license!!.name.toLowerCase()]
+            val detail = licenseInfo[license.license!!.name.lowercase(Locale.getDefault())]
             if (detail == null) {
                 incLicenseOverview(overview, LicenseNature.UNKNOWN.natureName)
                 continue

diff --git a/...c/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/dao/CveSecItemDao.kt b/...c/main/kotlin/com/tencent/bkrepo/analyst/component/manager/arrowhead/dao/CveSecItemDao.kt
@@ -36,6 +36,7 @@ import com.tencent.bkrepo.analyst.pojo.request.LoadResultArguments
 import org.springframework.data.mongodb.core.query.Criteria
 import org.springframework.data.mongodb.core.query.inValues
 import org.springframework.stereotype.Repository
+import java.util.Locale
 
 @Repository
 class CveSecItemDao : ResultItemDao<TCveSecItem>() {
@@ -63,7 +64,7 @@ class CveSecItemDao : ResultItemDao<TCveSecItem>() {
                 continue
             }
 
-            val prefix = vulId.substring(0, indexOfDash).toLowerCase()
+            val prefix = vulId.substring(0, indexOfDash).lowercase(Locale.getDefault())
             when (prefix) {
                 "cve" -> cveIds.add(vulId)
                 "cnnvd" -> cnnvdIds.add(vulId)

diff --git a/...src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/dependencycheck/Converter.kt b/...src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/dependencycheck/Converter.kt
@@ -38,6 +38,7 @@ import com.tencent.bkrepo.analyst.component.manager.dependencycheck.model.TDepen
 import com.tencent.bkrepo.analyst.component.manager.dependencycheck.model.TDependencyItemData
 import com.tencent.bkrepo.analyst.component.manager.knowledgebase.TCve
 import java.time.LocalDateTime
+import java.util.Locale
 
 object Converter {
     /**
@@ -171,5 +172,5 @@ object Converter {
         }
     }
 
-    fun pocIdOf(cveId: String) = "${DependencyScanner.TYPE.toLowerCase()}-$cveId"
+    fun pocIdOf(cveId: String) = "${DependencyScanner.TYPE.lowercase(Locale.getDefault())}-$cveId"
 }
diff --git a/...rc/main/kotlin/com/tencent/bkrepo/analyst/component/manager/standard/StandardConverter.kt b/...rc/main/kotlin/com/tencent/bkrepo/analyst/component/manager/standard/StandardConverter.kt
@@ -54,17 +54,18 @@ import com.tencent.bkrepo.common.api.pojo.Page
 import com.tencent.bkrepo.common.mongo.dao.util.Pages
 import com.tencent.bkrepo.common.query.model.PageLimit
 import org.springframework.stereotype.Component
+import java.util.Locale
 
 @Component("${StandardScanner.TYPE}Converter")
 class StandardConverter(private val licenseService: SpdxLicenseService) : ScannerConverter {
     @Suppress("UNCHECKED_CAST")
     override fun convertLicenseResult(result: Any): Page<FileLicensesResultDetail> {
         result as Page<LicenseResult>
         val licenseIds = result.records.map { it.licenseName }.distinct()
-        val licenses = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.toLowerCase() }
+        val licenses = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.lowercase(Locale.getDefault()) }
 
         val reports = result.records.map {
-            val detail = licenses[it.licenseName.toLowerCase()]
+            val detail = licenses[it.licenseName.lowercase(Locale.getDefault())]
             FileLicensesResultDetail(
                 licenseId = it.licenseName,
                 fullName = detail?.name ?: "",
@@ -140,7 +141,7 @@ class StandardConverter(private val licenseService: SpdxLicenseService) : Scanne
 
         // security统计
         securityResults?.forEach { securityResult ->
-            val severityLevel = Level.valueOf(securityResult.severity.toUpperCase()).level
+            val severityLevel = Level.valueOf(securityResult.severity.uppercase(Locale.getDefault())).level
             val shouldIgnore = filterRule?.shouldIgnore(
                 securityResult.vulId,
                 securityResult.cveId,
@@ -164,8 +165,8 @@ class StandardConverter(private val licenseService: SpdxLicenseService) : Scanne
             return overview
         }
 
-        val licenseIds = licenseResults.map { it.licenseName.toLowerCase() }.distinct()
-        val licensesInfo = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.toLowerCase() }
+        val licenseIds = licenseResults.map { it.licenseName.lowercase(Locale.getDefault()) }.distinct()
+        val licensesInfo = licenseService.listLicenseByIds(licenseIds).mapKeys { it.key.lowercase(Locale.getDefault()) }
 
         overview[LicenseOverviewKey.overviewKeyOf(TOTAL)] = licenseResults.size.toLong()
         for (licenseResult in licenseResults) {
@@ -175,7 +176,7 @@ class StandardConverter(private val licenseService: SpdxLicenseService) : Scanne
                 continue
             }
 
-            val detail = licensesInfo[licenseResult.licenseName.toLowerCase()]
+            val detail = licensesInfo[licenseResult.licenseName.lowercase(Locale.getDefault())]
             if (detail == null) {
                 incLicenseOverview(overview, LicenseNature.UNKNOWN.natureName)
                 continue

diff --git a/...z-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/Converter.kt b/...z-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/Converter.kt
@@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.analysis.pojo.scanner.trivy.VulnerabilityItem
 import com.tencent.bkrepo.repository.constant.SYSTEM_USER
 import com.tencent.bkrepo.analyst.component.manager.knowledgebase.TCve
 import java.time.LocalDateTime
+import java.util.Locale
 
 object Converter {
     /**
@@ -63,5 +64,6 @@ object Converter {
         }
     }
 
-    private fun pocIdOf(vulnerabilityId: String) = "${TrivyScanner.TYPE.toLowerCase()}-$vulnerabilityId"
+    private fun pocIdOf(vulnerabilityId: String) =
+        "${TrivyScanner.TYPE.lowercase(Locale.getDefault())}-$vulnerabilityId"
 }
diff --git a/...lyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/TrivyConverter.kt b/...lyst/src/main/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/TrivyConverter.kt
@@ -44,6 +44,7 @@ import com.tencent.bkrepo.analyst.pojo.response.ArtifactVulnerabilityInfo
 import com.tencent.bkrepo.analyst.utils.ScanPlanConverter
 import org.springframework.data.domain.PageRequest
 import org.springframework.stereotype.Component
+import java.util.Locale
 
 @Component("${TrivyScanner.TYPE}Converter")
 class TrivyConverter : ScannerConverter {
@@ -54,7 +55,7 @@ class TrivyConverter : ScannerConverter {
         val reports = result.records.mapTo(HashSet(result.records.size)) {
             ArtifactVulnerabilityInfo(
                 vulId = it.data.vulnerabilityId,
-                severity = ScanPlanConverter.convertToLeakLevel(it.data.severity.toLowerCase()),
+                severity = ScanPlanConverter.convertToLeakLevel(it.data.severity.lowercase(Locale.getDefault())),
                 pkgName = it.data.pkgName,
                 installedVersion = setOf(it.data.installedVersion),
                 title = it.data.title,
@@ -82,9 +83,9 @@ class TrivyConverter : ScannerConverter {
         // cve count
         scanExecutorResult.vulnerabilityItems.forEach {
             if (it.severity == "UNKNOWN") {
-                it.severity = Level.CRITICAL.levelName.toUpperCase()
+                it.severity = Level.CRITICAL.levelName.uppercase(Locale.getDefault())
             }
-            val overviewKey = CveOverviewKey.overviewKeyOf(it.severity.toLowerCase())
+            val overviewKey = CveOverviewKey.overviewKeyOf(it.severity.lowercase(Locale.getDefault()))
             overview[overviewKey] = overview.getOrDefault(overviewKey, 0L) + 1L
         }
         return overview

diff --git a/...ain/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/dao/VulnerabilityItemDao.kt b/...ain/kotlin/com/tencent/bkrepo/analyst/component/manager/trivy/dao/VulnerabilityItemDao.kt
@@ -35,13 +35,15 @@ import com.tencent.bkrepo.analyst.pojo.request.trivy.TrivyLoadResultArguments
 import org.springframework.data.mongodb.core.query.Criteria
 import org.springframework.data.mongodb.core.query.inValues
 import org.springframework.stereotype.Repository
+import java.util.Locale
 
 @Repository
 class VulnerabilityItemDao : ResultItemDao<TVulnerabilityItem>() {
     override fun customizePageBy(criteria: Criteria, arguments: LoadResultArguments): Criteria {
         require(arguments is TrivyLoadResultArguments)
         if (arguments.vulnerabilityLevels.isNotEmpty()) {
-            val levels = arguments.vulnerabilityLevels + arguments.vulnerabilityLevels.map { it.toUpperCase() }
+            val levels =
+                arguments.vulnerabilityLevels + arguments.vulnerabilityLevels.map { it.uppercase(Locale.getDefault()) }
             criteria.and(dataKey(VulnerabilityItem::severity.name)).inValues(levels)
         }
         if (arguments.vulIds.isNotEmpty()) {