feat: 节点列表拉取时查询节点总个数逻辑调整TencentBlueKing#2532

* feat: 节点列表拉取时查询节点总个数逻辑调整TencentBlueKing#2532 * feat: 节点列表拉取时查询节点总个数逻辑调整TencentBlueKing#2532 * feat: 当超过一次性拉取节点个数限制时，只查询该限制数量的节点TencentBlueKing#2532
yaoxuwan · Sep 20, 2024 · 95632ae · 95632ae
1 parent 785f444
commit 95632ae
Show file tree

Hide file tree

Showing 6 changed files with 72 additions and 12 deletions.
diff --git a/...z-repository/src/main/kotlin/com/tencent/bkrepo/repository/config/RepositoryProperties.kt b/...z-repository/src/main/kotlin/com/tencent/bkrepo/repository/config/RepositoryProperties.kt
@@ -53,5 +53,10 @@ data class RepositoryProperties(
     /**
      * 用于验证bkci webhook签名
      */
-    var bkciWebhookSecret: String = ""
+    var bkciWebhookSecret: String = "",
+    /**
+     * 当目录节点上的num字段小于该值时，去db中实时count目录大小
+     * 注意： 此配置的值要比listCountLimit大
+     */
+    var subNodeLimit: Long = 100000000L
 )
diff --git a/...sitory/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/...sitory/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt
@@ -70,6 +70,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateReque
 import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest
 import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.repository.service.node.NodeService
+import com.tencent.bkrepo.repository.service.repo.ProjectService
 import com.tencent.bkrepo.repository.service.repo.QuotaService
 import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
 import com.tencent.bkrepo.repository.util.MetadataUtils
@@ -105,7 +106,8 @@ abstract class NodeBaseService(
     open val servicePermissionClient: ServicePermissionClient,
     open val routerControllerClient: RouterControllerClient,
     open val routerControllerProperties: RouterControllerProperties,
-    open val blockNodeService: BlockNodeService
+    open val blockNodeService: BlockNodeService,
+    open val projectService: ProjectService,
 ) : NodeService {
 
     @Autowired
@@ -126,9 +128,11 @@ abstract class NodeBaseService(
             val (hasPermissionPaths, noPermissionPaths) = getPermissionPaths(userId, projectId, repoName)
             option.hasPermissionPath = hasPermissionPaths
             option.noPermissionPath = noPermissionPaths
-            val query = NodeQueryHelper.nodeListQuery(projectId, repoName, getArtifactFullPath(), option)
-            if (nodeDao.count(query) > repositoryProperties.listCountLimit) {
-                throw ErrorCodeException(ArtifactMessageCode.NODE_LIST_TOO_LARGE)
+            var query = NodeQueryHelper.nodeListQuery(projectId, repoName, getArtifactFullPath(), option)
+            val totalNum = getTotalNodeNum(artifact, query)
+            if (totalNum > repositoryProperties.listCountLimit) {
+                val pageRequest = Pages.ofRequest(1, repositoryProperties.listCountLimit.toInt())
+                query = query.with(pageRequest)
             }
             return nodeDao.find(query).map { convert(it)!! }
         }
@@ -146,10 +150,9 @@ abstract class NodeBaseService(
             Preconditions.checkArgument(pageNumber >= 0, "pageNumber")
             Preconditions.checkArgument(pageSize >= 0 && pageSize <= repositoryProperties.listCountLimit, "pageSize")
             val query = NodeQueryHelper.nodeListQuery(projectId, repoName, getArtifactFullPath(), option)
-            val totalRecords = nodeDao.count(query)
+            val totalRecords = getTotalNodeNum(artifact, query)
             val pageRequest = Pages.ofRequest(pageNumber, pageSize)
             val records = nodeDao.find(query.with(pageRequest)).map { convert(it)!! }
-
             return Pages.ofResponse(pageRequest, totalRecords, records)
         }
     }
@@ -264,6 +267,46 @@ abstract class NodeBaseService(
         }
     }
 
+    private fun getTotalNodeNum(artifact: ArtifactInfo, query: Query): Long {
+        // 避免当目录下节点过多去进行count产生慢查询，使用目录对应的子节点个数进行判断
+        // 只有节点个数小于配置的大小时，才去实时获取对应总节点个数
+        val subNodeNum = getSubNodeNum(artifact)
+        val limit = if (repositoryProperties.listCountLimit > repositoryProperties.subNodeLimit) {
+            repositoryProperties.listCountLimit
+        } else {
+            repositoryProperties.subNodeLimit
+        }
+        return if (subNodeNum <= limit) {
+            nodeDao.count(query)
+        } else {
+            // *2主要是因为subnodeNum不包含目录
+            subNodeNum * 2
+        }
+    }
+
+    /**
+     * 获取该节点下的子节点（不包含目录）个数
+     */
+    private fun getSubNodeNum(artifact: ArtifactInfo): Long {
+        with(artifact) {
+            val fullPath = artifact.getArtifactFullPath()
+            if (PathUtils.isRoot(fullPath)) {
+                return try {
+                    projectService.getProjectMetricsInfo(artifact.projectId)?.repoMetrics?.firstOrNull {
+                        it.repoName == artifact.repoName
+                    }?.num ?: -1
+                } catch (e: Exception) {
+                    -1
+                }
+            } else {
+                val node = nodeDao.findNode(projectId, repoName, fullPath) ?: return 0
+                if (!node.folder) return 0
+                if (node.nodeNum == null) return -1
+                return node.nodeNum!!
+            }
+        }
+    }
+
     private fun afterCreate(repo: TRepository, node: TNode) {
         with(node) {
             if (isGenericRepo(repo)) {

diff --git a/...sitory/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/...sitory/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt
@@ -52,6 +52,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest
 import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest
 import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest
 import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest
+import com.tencent.bkrepo.repository.service.repo.ProjectService
 import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.repository.service.repo.QuotaService
 import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
@@ -77,6 +78,7 @@ class NodeServiceImpl(
     override val routerControllerClient: RouterControllerClient,
     override val routerControllerProperties: RouterControllerProperties,
     override val blockNodeService: BlockNodeService,
+    override val projectService: ProjectService,
     private val archiveClient: ArchiveClient,
 ) : NodeBaseService(
     nodeDao,
@@ -90,7 +92,8 @@ class NodeServiceImpl(
     servicePermissionClient,
     routerControllerClient,
     routerControllerProperties,
-    blockNodeService
+    blockNodeService,
+    projectService
 ) {
 
     override fun computeSize(

diff --git a/...com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt b/...com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt
@@ -59,6 +59,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest
 import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.repository.service.node.impl.NodeRestoreSupport
 import com.tencent.bkrepo.repository.service.node.impl.NodeServiceImpl
+import com.tencent.bkrepo.repository.service.repo.ProjectService
 import com.tencent.bkrepo.repository.service.repo.QuotaService
 import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
 import com.tencent.bkrepo.repository.util.NodeQueryHelper
@@ -85,6 +86,7 @@ class CommitEdgeCenterNodeServiceImpl(
     override val routerControllerClient: RouterControllerClient,
     override val routerControllerProperties: RouterControllerProperties,
     override val blockNodeService: BlockNodeService,
+    override val projectService: ProjectService,
     val clusterProperties: ClusterProperties,
     val archiveClient: ArchiveClient,
 ) : NodeServiceImpl(
@@ -100,7 +102,8 @@ class CommitEdgeCenterNodeServiceImpl(
     routerControllerClient,
     routerControllerProperties,
     blockNodeService,
-    archiveClient
+    projectService,
+    archiveClient,
 ) {
 
     override fun checkRepo(projectId: String, repoName: String): TRepository {

diff --git a/...c/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt b/...c/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt
@@ -46,6 +46,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateReque
 import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest
 import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService
+import com.tencent.bkrepo.repository.service.repo.ProjectService
 import com.tencent.bkrepo.repository.service.repo.QuotaService
 import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
 import com.tencent.bkrepo.router.api.RouterControllerClient
@@ -63,7 +64,8 @@ abstract class EdgeNodeBaseService(
     override val servicePermissionClient: ServicePermissionClient,
     override val routerControllerProperties: RouterControllerProperties,
     override val blockNodeService: BlockNodeService,
-    open val clusterProperties: ClusterProperties
+    override val projectService: ProjectService,
+    open val clusterProperties: ClusterProperties,
 ) : NodeBaseService(
     nodeDao,
     repositoryDao,
@@ -76,7 +78,8 @@ abstract class EdgeNodeBaseService(
     servicePermissionClient,
     routerControllerClient,
     routerControllerProperties,
-    blockNodeService
+    blockNodeService,
+    projectService
 ) {
 
     val centerNodeClient: ClusterNodeClient by lazy {

diff --git a/...c/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/...c/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt
@@ -65,6 +65,7 @@ import com.tencent.bkrepo.repository.service.node.impl.NodeMoveCopySupport
 import com.tencent.bkrepo.repository.service.node.impl.NodeRenameSupport
 import com.tencent.bkrepo.repository.service.node.impl.NodeRestoreSupport
 import com.tencent.bkrepo.repository.service.node.impl.NodeStatsSupport
+import com.tencent.bkrepo.repository.service.repo.ProjectService
 import com.tencent.bkrepo.repository.service.repo.QuotaService
 import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
 import com.tencent.bkrepo.router.api.RouterControllerClient
@@ -90,6 +91,7 @@ class EdgeNodeServiceImpl(
     override val routerControllerClient: RouterControllerClient,
     override val routerControllerProperties: RouterControllerProperties,
     override val blockNodeService: BlockNodeService,
+    override val projectService: ProjectService,
     val archiveClient: ArchiveClient,
 ) : EdgeNodeBaseService(
     nodeDao,
@@ -104,7 +106,8 @@ class EdgeNodeServiceImpl(
     servicePermissionClient,
     routerControllerProperties,
     blockNodeService,
-    clusterProperties,
+    projectService,
+    clusterProperties
 ) {
     override fun computeSize(
         artifact: ArtifactInfo,