From 409062acf491db562a40c1d08b1bd9e65670d16f Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:51:03 +0800 Subject: [PATCH 01/11] =?UTF-8?q?feat:=20=E5=88=A0=E9=99=A4=E5=B7=B2?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E6=96=87=E4=BB=B6=E9=80=BB=E8=BE=91=E8=B0=83?= =?UTF-8?q?=E6=95=B4#2425?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 节点清理逻辑保持一致#2425 * feat: 删除已存在文件逻辑调整#2425 * feat: commitedge模式下的节点删除相关逻辑调整#2425 * feat: 节点移动逻辑调整#2425 * feat: 代码调整#2425 * feat: 命名调整#2425 --- .../service/node/NodeDeleteOperation.kt | 7 ++ .../service/node/impl/NodeBaseService.kt | 6 +- .../service/node/impl/NodeDeleteSupport.kt | 65 +++++++++++++------ .../service/node/impl/NodeMoveCopySupport.kt | 5 +- .../service/node/impl/NodeServiceImpl.kt | 14 +++- .../CommitEdgeCenterNodeDeleteSupport.kt | 13 ++-- .../center/CommitEdgeCenterNodeServiceImpl.kt | 19 ++++-- .../node/impl/edge/EdgeNodeServiceImpl.kt | 8 +++ 8 files changed, 102 insertions(+), 35 deletions(-) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeDeleteOperation.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeDeleteOperation.kt index 3679cf5ef5..d156522974 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeDeleteOperation.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeDeleteOperation.kt @@ -56,6 +56,13 @@ interface NodeDeleteOperation { */ fun countDeleteNodes(nodesDeleteRequest: NodesDeleteRequest): Long + /** + * 根据fullpath删除对应节点 + * 不会进行已删除节点数据返回 + * 不会进行容量清理，需要自行进行容量清理 + */ + fun deleteByFullPathWithoutDecreaseVolume(projectId: String, repoName: String, fullPath: String, operator: String) + /** * 根据全路径删除文件或者目录 */ diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt index ce53f37a32..36d9573364 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt @@ -392,7 +392,7 @@ abstract class NodeBaseService( return nodes } - open fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String): LocalDateTime? { + open fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { with(createRequest) { val existNode = nodeDao.findNode(projectId, repoName, fullPath) if (existNode != null) { @@ -403,12 +403,12 @@ abstract class NodeBaseService( } else { val changeSize = this.size?.minus(existNode.size) ?: -existNode.size quotaService.checkRepoQuota(projectId, repoName, changeSize) - return deleteByPath(projectId, repoName, fullPath, operator).deletedTime + deleteByFullPathWithoutDecreaseVolume(projectId, repoName, fullPath, operator) + quotaService.decreaseUsedVolume(projectId, repoName, existNode.size) } } else { quotaService.checkRepoQuota(projectId, repoName, this.size ?: 0) } - return null } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeDeleteSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeDeleteSupport.kt index 8dbcc0d50a..da26725be1 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeDeleteSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeDeleteSupport.kt @@ -114,22 +114,24 @@ open class NodeDeleteSupport( } } + override fun deleteByFullPathWithoutDecreaseVolume( + projectId: String, + repoName: String, + fullPath: String, + operator: String + ) { + val criteria = buildCriteria(projectId, repoName, fullPath) + val query = Query(criteria) + delete(query, operator, criteria, projectId, repoName, listOf(fullPath), false) + } + override fun deleteByPath( projectId: String, repoName: String, fullPath: String, operator: String ): NodeDeleteResult { - val normalizedFullPath = PathUtils.normalizeFullPath(fullPath) - val normalizedPath = PathUtils.toPath(normalizedFullPath) - val escapedPath = PathUtils.escapeRegex(normalizedPath) - val criteria = where(TNode::projectId).isEqualTo(projectId) - .and(TNode::repoName).isEqualTo(repoName) - .and(TNode::deleted).isEqualTo(null) - .orOperator( - where(TNode::fullPath).regex("^$escapedPath"), - where(TNode::fullPath).isEqualTo(normalizedFullPath) - ) + val criteria = buildCriteria(projectId, repoName, fullPath) val query = Query(criteria) return delete(query, operator, criteria, projectId, repoName, listOf(fullPath)) } @@ -173,19 +175,40 @@ open class NodeDeleteSupport( .andOperator(timeCriteria) val query = Query(criteria) val nodeDeleteResult = delete(query, operator, criteria, projectId, repoName) - publishEvent(buildNodeCleanEvent( - projectId, repoName, path, operator, nodeDeleteResult.deletedTime.toString()) + publishEvent( + buildNodeCleanEvent( + projectId, repoName, path, operator, nodeDeleteResult.deletedTime.toString() + ) ) return nodeDeleteResult } + private fun buildCriteria( + projectId: String, + repoName: String, + fullPath: String, + ): Criteria { + val normalizedFullPath = PathUtils.normalizeFullPath(fullPath) + val normalizedPath = PathUtils.toPath(normalizedFullPath) + val escapedPath = PathUtils.escapeRegex(normalizedPath) + val criteria = where(TNode::projectId).isEqualTo(projectId) + .and(TNode::repoName).isEqualTo(repoName) + .and(TNode::deleted).isEqualTo(null) + .orOperator( + where(TNode::fullPath).regex("^$escapedPath"), + where(TNode::fullPath).isEqualTo(normalizedFullPath) + ) + return criteria + } + private fun delete( query: Query, operator: String, criteria: Criteria, projectId: String, repoName: String, - fullPaths: List? = null + fullPaths: List? = null, + decreaseVolume: Boolean = true ): NodeDeleteResult { var deletedNum = 0L var deletedSize = 0L @@ -203,14 +226,16 @@ open class NodeDeleteSupport( if (deletedNum == 0L) { return NodeDeleteResult(deletedNum, deletedSize, deleteTime) } - var deletedCriteria = criteria.and(TNode::deleted).isEqualTo(deleteTime) - fullPaths?.let { - // 节点删除接口返回的数据排除目录 - deletedCriteria = deletedCriteria.and(TNode::folder).isEqualTo(false) - deletedNum = nodeDao.count(Query(deletedCriteria)) + if (decreaseVolume) { + var deletedCriteria = criteria.and(TNode::deleted).isEqualTo(deleteTime) + fullPaths?.let { + // 节点删除接口返回的数据排除目录 + deletedCriteria = deletedCriteria.and(TNode::folder).isEqualTo(false) + deletedNum = nodeDao.count(Query(deletedCriteria)) + } + deletedSize = nodeBaseService.aggregateComputeSize(deletedCriteria) + quotaService.decreaseUsedVolume(projectId, repoName, deletedSize) } - deletedSize = nodeBaseService.aggregateComputeSize(deletedCriteria) - quotaService.decreaseUsedVolume(projectId, repoName, deletedSize) fullPaths?.forEach { if (routerControllerProperties.enabled) { routerControllerClient.removeNodes(projectId, repoName, it) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt index 2e4d4c0b3d..1a648ffbd4 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt @@ -180,7 +180,10 @@ open class NodeMoveCopySupport( // 文件 -> 文件 & 允许覆盖: 删除old if (existNode?.folder == false && overwrite) { quotaService.checkRepoQuota(existNode.projectId, existNode.repoName, node.size - existNode.size) - nodeBaseService.deleteByPath(existNode.projectId, existNode.repoName, existNode.fullPath, operator) + nodeBaseService.deleteByFullPathWithoutDecreaseVolume( + existNode.projectId, existNode.repoName, existNode.fullPath, operator + ) + quotaService.decreaseUsedVolume(existNode.projectId, existNode.repoName, existNode.size) } } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt index 23882d5646..98a7bd8351 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt @@ -125,6 +125,14 @@ class NodeServiceImpl( return NodeDeleteSupport(this).countDeleteNodes(nodesDeleteRequest) } + override fun deleteByFullPathWithoutDecreaseVolume( + projectId: String, repoName: String, fullPath: String, operator: String + ) { + return NodeDeleteSupport(this).deleteByFullPathWithoutDecreaseVolume( + projectId, repoName, fullPath, operator + ) + } + @Transactional(rollbackFor = [Throwable::class]) override fun deleteByPath( projectId: String, @@ -196,15 +204,15 @@ class NodeServiceImpl( } override fun archiveNode(nodeArchiveRequest: NodeArchiveRequest) { - return NodeArchiveSupport(this,archiveClient).archiveNode(nodeArchiveRequest) + return NodeArchiveSupport(this, archiveClient).archiveNode(nodeArchiveRequest) } override fun restoreNode(nodeArchiveRequest: NodeArchiveRequest) { - return NodeArchiveSupport(this,archiveClient).restoreNode(nodeArchiveRequest) + return NodeArchiveSupport(this, archiveClient).restoreNode(nodeArchiveRequest) } override fun restoreNode(nodeRestoreRequest: NodeArchiveRestoreRequest): List { - return NodeArchiveSupport(this,archiveClient).restoreNode(nodeRestoreRequest) + return NodeArchiveSupport(this, archiveClient).restoreNode(nodeRestoreRequest) } override fun compressedNode(nodeCompressedRequest: NodeCompressedRequest) { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeDeleteSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeDeleteSupport.kt index db80c1b166..812bf41032 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeDeleteSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeDeleteSupport.kt @@ -113,8 +113,12 @@ class CommitEdgeCenterNodeDeleteSupport( var deletedSize = 0L var deletedNum = 0L val option = NodeListOption(includeFolder = false, deep = true) + val timeCriteria = Criteria().orOperator( + Criteria().and(TNode::lastAccessDate).lt(date).and(TNode::lastModifiedDate).lt(date), + Criteria().and(TNode::lastAccessDate).`is`(null).and(TNode::lastModifiedDate).lt(date), + ) val criteria = NodeQueryHelper.nodeListCriteria(projectId, repoName, path, option) - .and(TNode::createdDate).lt(date) + .andOperator(timeCriteria) .and(TNode::clusterNames).isEqualTo(clusterName) val pageSize = 1 var queryCount: Int @@ -163,12 +167,12 @@ class CommitEdgeCenterNodeDeleteSupport( } subNodes = nodeDao.find(query) if (subNodes.isEmpty()) { - deletedNumber += super.deleteByPath( + super.deleteByFullPathWithoutDecreaseVolume( projectId = folder.projectId, repoName = folder.repoName, fullPath = folder.fullPath, operator = operator - ).deletedNumber + ) } return NodeDeleteResult(deletedNumber, deletedSize, LocalDateTime.now()) } @@ -183,7 +187,8 @@ class CommitEdgeCenterNodeDeleteSupport( val srcCluster = SecurityUtils.getClusterName() ?: clusterProperties.self.name.toString() node.clusterNames = node.clusterNames.orEmpty().minus(srcCluster) if (node.clusterNames.orEmpty().isEmpty()) { - super.deleteByPath(node.projectId, node.repoName, node.fullPath, operator) + super.deleteByFullPathWithoutDecreaseVolume(node.projectId, node.repoName, node.fullPath, operator) + quotaService.decreaseUsedVolume(node.projectId, node.repoName, node.size) } else { val query = NodeQueryHelper.nodeQuery(node.projectId, node.repoName, node.fullPath) val update = Update().pull(TNode::clusterNames.name, srcCluster) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt index f8731311af..1702ae0a57 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt @@ -36,8 +36,8 @@ import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties import com.tencent.bkrepo.common.artifact.util.ClusterUtils import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition +import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier import com.tencent.bkrepo.fs.server.api.FsNodeClient @@ -114,7 +114,7 @@ class CommitEdgeCenterNodeServiceImpl( return repo } - override fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String): LocalDateTime? { + override fun checkConflictAndQuota(createRequest: NodeCreateRequest, fullPath: String) { with(createRequest) { val existNode = nodeDao.findNode(projectId, repoName, fullPath) if (existNode != null) { @@ -126,15 +126,26 @@ class CommitEdgeCenterNodeServiceImpl( ClusterUtils.checkIsSrcCluster(existNode.clusterNames) val changeSize = this.size?.minus(existNode.size) ?: -existNode.size quotaService.checkRepoQuota(projectId, repoName, changeSize) - return deleteByPath(projectId, repoName, fullPath, operator).deletedTime + deleteByFullPathWithoutDecreaseVolume(projectId, repoName, fullPath, operator) + quotaService.decreaseUsedVolume(projectId, repoName, existNode.size) } } else { quotaService.checkRepoQuota(projectId, repoName, this.size ?: 0) } - return null } } + override fun deleteByFullPathWithoutDecreaseVolume( + projectId: String, repoName: String, fullPath: String, operator: String + ) { + return CommitEdgeCenterNodeDeleteSupport(this, clusterProperties).deleteByFullPathWithoutDecreaseVolume( + projectId, + repoName, + fullPath, + operator + ) + } + override fun buildTNode(request: NodeCreateRequest): TNode { val node = super.buildTNode(request) SecurityUtils.getClusterName()?.let { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt index 5272514e8e..493476f95c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -153,6 +153,14 @@ class EdgeNodeServiceImpl( return NodeDeleteSupport(this).countDeleteNodes(nodesDeleteRequest) } + override fun deleteByFullPathWithoutDecreaseVolume( + projectId: String, repoName: String, fullPath: String, operator: String + ) { + return NodeDeleteSupport(this).deleteByFullPathWithoutDecreaseVolume( + projectId, repoName, fullPath, operator + ) + } + @Transactional(rollbackFor = [Throwable::class]) override fun deleteByPath( projectId: String, From 1e18e8f2611f72d76a08e8d0aba395e9e982cecd Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Tue, 6 Aug 2024 16:07:27 +0800 Subject: [PATCH 02/11] =?UTF-8?q?feat=20:=20=E5=A2=9E=E5=8A=A0=E5=8D=95?= =?UTF-8?q?=E4=BD=93=E5=8A=9F=E8=83=BD=E6=A0=A1=E9=AA=8Caction=20#2459?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Create assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml * Update assembly-check.yml --- .github/workflows/assembly-check.yml | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/assembly-check.yml diff --git a/.github/workflows/assembly-check.yml b/.github/workflows/assembly-check.yml new file mode 100644 index 0000000000..e924e3cdc6 --- /dev/null +++ b/.github/workflows/assembly-check.yml @@ -0,0 +1,46 @@ +name: Build assembly application and test + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ master ] + paths: + - 'src/backend/**' + pull_request: + branches: [ master, release-* ] + paths: + - 'src/backend/**' +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build-and-test: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Cache Gradle + uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: Start MongoDB + run: | + docker run -d --name mongodb -p 27017:27017 mongo + - name: Gradle Build and test + working-directory: src/backend/ + run: | + ./gradlew clean + ./gradlew boot-assembly:build -x test + java -jar -Dlogging.path=boot-assembly/build/libs/bkrepo/logs -Dstorage.filesystem.path=boot-assembly/build/libs/bkrepo/bkrepo boot-assembly/build/libs/boot-assembly-1.0.0-RELEASE.jar & + sleep 60 + curl http://localhost:8080/helm/blueking/helm-local/index.yaml From 78ade12ae64a18f6850fd2fa1bc97c77dcd0deb7 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Tue, 6 Aug 2024 16:34:17 +0800 Subject: [PATCH 03/11] =?UTF-8?q?feat:=20=E8=A7=A3=E5=86=B3=E5=8D=95?= =?UTF-8?q?=E4=BD=93=E7=B1=BB=E5=8A=A0=E8=BD=BD=E5=86=B2=E7=AA=81#2461?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 解决单体类加载冲突#2461 * feat: 解决DdcMeterBinder循环依赖#2461 * feat: 解决BDZipManager循环依赖#2461 * feat: 解决循环依赖#2461 * feat: 解决单体类加载报错#2461 --- .../bkrepo/analyst/configuration/ScannerConfiguration.kt | 6 +++--- .../{ScanEventConsumer.kt => AnalystScanEventConsumer.kt} | 4 ++-- .../tencent/bkrepo/archive/core/archive/ArchiveManager.kt | 5 ++++- .../tencent/bkrepo/archive/core/compress/BDZipManager.kt | 6 +++++- .../kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt | 6 +++++- .../tencent/bkrepo/webhook/config/WebHookConsumerConfig.kt | 6 +++--- ...factEventConsumer.kt => WebhookArtifactEventConsumer.kt} | 4 ++-- 7 files changed, 24 insertions(+), 13 deletions(-) rename src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/{ScanEventConsumer.kt => AnalystScanEventConsumer.kt} (98%) rename src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/executor/{ArtifactEventConsumer.kt => WebhookArtifactEventConsumer.kt} (97%) diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt index 902b98c4a2..03e0c4b654 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt @@ -30,7 +30,7 @@ package com.tencent.bkrepo.analyst.configuration import com.tencent.bkrepo.analysis.executor.api.ExecutorClient import com.tencent.bkrepo.analyst.dispatcher.SubtaskDispatcherFactory import com.tencent.bkrepo.analyst.dispatcher.SubtaskPoller -import com.tencent.bkrepo.analyst.event.ScanEventConsumer +import com.tencent.bkrepo.analyst.event.AnalystScanEventConsumer import com.tencent.bkrepo.analyst.service.ExecutionClusterService import com.tencent.bkrepo.analyst.service.ScannerService import com.tencent.bkrepo.analyst.service.impl.OperateLogServiceImpl @@ -91,10 +91,10 @@ class ScannerConfiguration { @Bean("scanEventConsumer") fun scanEventConsumer( - scanEventConsumer: ScanEventConsumer + analystScanEventConsumer: AnalystScanEventConsumer ): Consumer { return Consumer { - scanEventConsumer.accept(it) + analystScanEventConsumer.accept(it) } } } diff --git a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/ScanEventConsumer.kt b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/AnalystScanEventConsumer.kt similarity index 98% rename from src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/ScanEventConsumer.kt rename to src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/AnalystScanEventConsumer.kt index aea9aa3757..d0bd3fccd7 100644 --- a/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/ScanEventConsumer.kt +++ b/src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/AnalystScanEventConsumer.kt @@ -63,7 +63,7 @@ import org.springframework.stereotype.Component * 对应binding name为scanEventConsumer-in-0 */ @Component -class ScanEventConsumer( +class AnalystScanEventConsumer( private val spdxLicenseService: SpdxLicenseService, private val scanService: ScanService, private val scannerService: ScannerService, @@ -287,6 +287,6 @@ class ScanEventConsumer( } companion object { - private val logger = LoggerFactory.getLogger(ScanEventConsumer::class.java) + private val logger = LoggerFactory.getLogger(AnalystScanEventConsumer::class.java) } } diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/archive/ArchiveManager.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/archive/ArchiveManager.kt index 2cd61c9812..115cd43037 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/archive/ArchiveManager.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/archive/ArchiveManager.kt @@ -39,7 +39,6 @@ import java.util.function.Function class ArchiveManager( private val archiveProperties: ArchiveProperties, private val fileProvider: PriorityFileProvider, - private val storageService: StorageService, ) : Function> { @Autowired @Lazy @@ -49,6 +48,10 @@ class ArchiveManager( @Lazy private lateinit var archiveFileRepository: ArchiveFileRepository + @Autowired + @Lazy + private lateinit var storageService: StorageService + private val tika = Tika() private val compressPool = ArchiveUtils.newFixedAndCachedThreadPool( archiveProperties.compress.compressThreads, diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt index 55f29c148b..237bc9e393 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt @@ -42,7 +42,6 @@ import java.util.function.Function @Component class BDZipManager( private val archiveProperties: ArchiveProperties, - private val storageService: StorageService, private val fileProvider: PriorityFileProvider, ) : Function> { @Autowired @@ -53,6 +52,11 @@ class BDZipManager( @Lazy private lateinit var fileReferenceClient: FileReferenceClient + @Autowired + @Lazy + private lateinit var storageService: StorageService + + @Autowired @Lazy private lateinit var compressFileRepository: CompressFileRepository diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt index ca82e52e8f..843e0159de 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt @@ -34,7 +34,7 @@ import io.micrometer.core.instrument.binder.MeterBinder import org.springframework.stereotype.Component @Component -class DdcMeterBinder(private val registry: MeterRegistry) : MeterBinder { +class DdcMeterBinder : MeterBinder { /** * ref inline加载耗时 */ @@ -65,7 +65,11 @@ class DdcMeterBinder(private val registry: MeterRegistry) : MeterBinder { */ lateinit var legacyRefStoreTimer: Timer + private lateinit var registry: MeterRegistry + + override fun bindTo(registry: MeterRegistry) { + this.registry = registry refInlineLoadTimer = Timer .builder(DDC_REF_LOAD) .tag("inline", "true") diff --git a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/config/WebHookConsumerConfig.kt b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/config/WebHookConsumerConfig.kt index 69c3fb482b..2cfc9438c3 100644 --- a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/config/WebHookConsumerConfig.kt +++ b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/config/WebHookConsumerConfig.kt @@ -32,7 +32,7 @@ package com.tencent.bkrepo.webhook.config import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent -import com.tencent.bkrepo.webhook.executor.ArtifactEventConsumer +import com.tencent.bkrepo.webhook.executor.WebhookArtifactEventConsumer import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.messaging.Message @@ -43,10 +43,10 @@ class WebHookConsumerConfig { @Bean("artifactEventWebhook") fun artifactEventConsumer( - artifactEventConsumer: ArtifactEventConsumer + webhookArtifactEventConsumer: WebhookArtifactEventConsumer ): Consumer> { return Consumer { - artifactEventConsumer.accept(it) + webhookArtifactEventConsumer.accept(it) } } } \ No newline at end of file diff --git a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/executor/ArtifactEventConsumer.kt b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/executor/WebhookArtifactEventConsumer.kt similarity index 97% rename from src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/executor/ArtifactEventConsumer.kt rename to src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/executor/WebhookArtifactEventConsumer.kt index c452d61dc0..cf53776ed4 100644 --- a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/executor/ArtifactEventConsumer.kt +++ b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/executor/WebhookArtifactEventConsumer.kt @@ -49,7 +49,7 @@ import java.util.regex.Pattern * 事件消息消费者 */ @Component -class ArtifactEventConsumer( +class WebhookArtifactEventConsumer( private val webHookDao: WebHookDao, private val webHookExecutor: WebHookExecutor, private val webHookProperties: WebHookProperties @@ -140,6 +140,6 @@ class ArtifactEventConsumer( } companion object { - private val logger = LoggerFactory.getLogger(ArtifactEventConsumer::class.java) + private val logger = LoggerFactory.getLogger(WebhookArtifactEventConsumer::class.java) } } From 88d3dfc3b2bd31e12428c2b3980880c6e856360e Mon Sep 17 00:00:00 2001 From: owen Date: Wed, 7 Aug 2024 10:08:16 +0800 Subject: [PATCH 04/11] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E5=90=8D=E7=A7=B0=20#2421?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/bkrepo/auth/model/TRepoAuthConfig.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TRepoAuthConfig.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TRepoAuthConfig.kt index 1c6bb21b52..09ba910e8d 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TRepoAuthConfig.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TRepoAuthConfig.kt @@ -40,7 +40,7 @@ import java.time.LocalDateTime @Document("repo_auth_mode") @CompoundIndexes( CompoundIndex(name = "repo_idx", def = "{'projectId': 1, 'repoName': 1}", background = true, unique = true), - CompoundIndex(name = "access_ctrl_idx", def = "{'accessControlMode': 1}", background = true) + CompoundIndex(name = "access_ctrl_mode_idx", def = "{'accessControlMode': 1}", background = true) ) data class TRepoAuthConfig( var id: String? = null, From 13b41da9a453b9f879b3d053671be8af07f90b69 Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Wed, 7 Aug 2024 11:01:20 +0800 Subject: [PATCH 05/11] =?UTF-8?q?feat=20:=20Release=201.5.1=E5=90=88?= =?UTF-8?q?=E5=B9=B6=E5=88=B0master=20#2467?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: fs-server移除对generic remote仓库的支持 #2411 * feat: 增加CachedFileWriter日志 #2451 * feat: 构件传输记录指标增加userId #2448 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat : 修复权限校验bug #2375 * feat: Commit-Edge模式兼容独立集群模式 #2298 * feat: 添加 node_download 类型 #2380 * feat: 添加 node_download 类型 #2380 * feat: 添加 download 类型 #2380 * feat : 修复权限校验bug #2375 * feat: Commit-Edge模式兼容独立集群模式 #2298 * feat: 构件传输记录指标增加userId #2448 * feat: 构件传输记录指标增加userId #2448 * feat: 构件传输记录指标增加userId #2448 --------- Co-authored-by: owen Co-authored-by: zacYL <100330102+zacYL@users.noreply.github.com> --------- Co-authored-by: owen Co-authored-by: zacYL <100330102+zacYL@users.noreply.github.com> --- .../service/bkdevops/DevopsPermissionServiceImpl.kt | 2 +- .../event/listener/ArtifactTransferListener.kt | 10 +++++++--- .../common/artifact/metrics/ArtifactTransferRecord.kt | 2 ++ .../artifact/metrics/ArtifactTransferRecordLog.kt | 1 + .../artifact/metrics/export/ArtifactMetricsExporter.kt | 1 + .../common/storage/core/cache/CachedFileWriter.kt | 8 ++++++-- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt index a77423d0dd..3bcc86e6c6 100644 --- a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt +++ b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt @@ -305,4 +305,4 @@ class DevopsPermissionServiceImpl constructor( companion object { private val logger = LoggerFactory.getLogger(DevopsPermissionServiceImpl::class.java) } -} \ No newline at end of file +} diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt index ee3132c531..b923f179d2 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt @@ -37,6 +37,7 @@ import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY import com.tencent.bkrepo.common.artifact.event.ArtifactReceivedEvent import com.tencent.bkrepo.common.artifact.event.ArtifactResponseEvent import com.tencent.bkrepo.common.artifact.event.ChunkArtifactTransferEvent +import com.tencent.bkrepo.common.artifact.hash.md5 import com.tencent.bkrepo.common.artifact.metrics.ArtifactCacheMetrics import com.tencent.bkrepo.common.artifact.metrics.ArtifactMetrics import com.tencent.bkrepo.common.artifact.metrics.ArtifactMetricsProperties @@ -106,7 +107,8 @@ class ArtifactTransferListener( host = artifactMetricsProperties.host, builderAgentList = artifactMetricsProperties.builderAgentList, clientAgentList = artifactMetricsProperties.clientAgentList - ).name + ).name, + userId = SecurityUtils.getUserId().md5() ) if (SecurityUtils.getUserId() != SYSTEM_USER) { projectUsageStatisticsService.inc(projectId = projectId, receivedBytes = throughput.bytes) @@ -151,7 +153,8 @@ class ArtifactTransferListener( host = artifactMetricsProperties.host, builderAgentList = artifactMetricsProperties.builderAgentList, clientAgentList = artifactMetricsProperties.clientAgentList - ).name + ).name, + userId = SecurityUtils.getUserId().md5() ) if (SecurityUtils.getUserId() != SYSTEM_USER) { projectUsageStatisticsService.inc(projectId = projectId, responseBytes = throughput.bytes) @@ -207,7 +210,8 @@ class ArtifactTransferListener( host = artifactMetricsProperties.host, builderAgentList = artifactMetricsProperties.builderAgentList, clientAgentList = artifactMetricsProperties.clientAgentList - ).name + ).name, + userId = SecurityUtils.getUserId().md5() ) if (artifactMetricsProperties.collectByLog) { logger.info( diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecord.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecord.kt index 04d8e2e2dd..95809747b1 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecord.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecord.kt @@ -63,6 +63,8 @@ data class ArtifactTransferRecord( val fullPath: String, @Column(name = "agent") val agent: String, + @Column(name = "userId") + val userId: String, ) { companion object { const val RECEIVE = "RECEIVE" diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecordLog.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecordLog.kt index efb173f3f6..2136727fa0 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecordLog.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecordLog.kt @@ -46,6 +46,7 @@ class ArtifactTransferRecordLog( val repoName: String = record.repoName val agent: String = record.agent val fullPath: String = record.fullPath + val userId: String = record.userId val service: String? = commonTag["service"] val instance: String? = commonTag["instance"] val host: String? = commonTag["host"] diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/export/ArtifactMetricsExporter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/export/ArtifactMetricsExporter.kt index a74f99c36c..fc2d6a08ea 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/export/ArtifactMetricsExporter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/export/ArtifactMetricsExporter.kt @@ -74,6 +74,7 @@ class ArtifactMetricsExporter( labels[ArtifactTransferRecord::elapsed.name] = record.elapsed.toString() labels[ArtifactTransferRecord::type.name] = record.type labels[ArtifactTransferRecord::agent.name] = record.agent + labels[ArtifactTransferRecord::userId.name] = record.userId return labels } diff --git a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CachedFileWriter.kt b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CachedFileWriter.kt index c67f68c6d7..f19dece401 100644 --- a/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CachedFileWriter.kt +++ b/src/backend/common/common-storage/storage-service/src/main/kotlin/com/tencent/bkrepo/common/storage/core/cache/CachedFileWriter.kt @@ -80,6 +80,7 @@ class CachedFileWriter( } } catch (ignore: FileAlreadyExistsException) { // 如果目录或者文件已存在则忽略 + logger.info("initial temp file $tempFilePath failed: ${ignore.message}") } catch (exception: Exception) { logger.error("initial CacheFileWriter error: $exception", exception) close() @@ -92,6 +93,7 @@ class CachedFileWriter( it.write(i) } catch (ignored: Exception) { // ignored + logger.info("write data to temp file $tempFilePath failed: ${ignored.message}") close() } } @@ -103,6 +105,7 @@ class CachedFileWriter( it.write(buffer, off, length) } catch (ignored: Exception) { // ignored + logger.info("write data to temp file $tempFilePath failed: ${ignored.message}") close() } } @@ -172,12 +175,13 @@ class CachedFileWriter( if (!cacheFilePath.existReal()) { Files.move(tempFilePath, cacheFilePath) listener?.onCacheFileWritten(filename, cacheFilePath) - logger.info("Success cache file $filename") + logger.info("Success cache file $filename, move temp file $tempFilePath to cache file $cacheFilePath") } } catch (ignore: FileAlreadyExistsException) { logger.info("File[$cacheFilePath] already exists") } catch (exception: Exception) { - logger.error("Finish CacheFileWriter error: $exception", exception) + logger.error("Finish CacheFileWriter error, " + + "move temp file $tempFilePath to cache file $cacheFilePath: $exception", exception) } } From 4b52aabdc9bd795f76bab8ee1e4abbba7e665b20 Mon Sep 17 00:00:00 2001 From: lannoy0523 <46735290+lannoy0523@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:47:53 +0800 Subject: [PATCH 06/11] =?UTF-8?q?bug:=20=E4=BC=98=E5=8C=96op=E7=99=BB?= =?UTF-8?q?=E5=BD=95=EF=BC=88=E5=A4=AA=E6=B9=96=E7=99=BB=E5=BD=95=E6=83=85?= =?UTF-8?q?=E5=86=B5=E4=B8=8B=EF=BC=8C=E6=97=A0cookie=E6=AD=A3=E5=B8=B8?= =?UTF-8?q?=E7=99=BB=E5=BD=95=EF=BC=89=20#2441?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bug: 优化op登录（太湖登录情况下，无cookie正常登录） #2441 * bug: 优化op登录（太湖登录情况下，无cookie正常登录） #2441 --- .../devops-op/src/layout/components/Navbar.vue | 2 +- src/frontend/devops-op/src/permission.js | 18 +++++++++++++++--- .../devops-op/src/store/modules/user.js | 5 +++-- src/frontend/devops-op/src/utils/auth.js | 10 ++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/frontend/devops-op/src/layout/components/Navbar.vue b/src/frontend/devops-op/src/layout/components/Navbar.vue index e31523d615..39cc2a720b 100644 --- a/src/frontend/devops-op/src/layout/components/Navbar.vue +++ b/src/frontend/devops-op/src/layout/components/Navbar.vue @@ -43,7 +43,7 @@ export default { }, data() { return { - userName: localStorage.getItem('userName') || getBkUid() + userName: localStorage.getItem('userName') } }, computed: { diff --git a/src/frontend/devops-op/src/permission.js b/src/frontend/devops-op/src/permission.js index aa1cf764b4..1979696fff 100644 --- a/src/frontend/devops-op/src/permission.js +++ b/src/frontend/devops-op/src/permission.js @@ -3,8 +3,8 @@ import store from './store' import { Message } from 'element-ui' import NProgress from 'nprogress' // progress bar import 'nprogress/nprogress.css' // progress bar style -import { getBkUid, getToken } from '@/utils/auth' // get token from cookie -import { toLoginPage } from '@/utils/login' +import { getBkUid, getTempUid, getToken, setTempUid } from '@/utils/auth' // get token from cookie +import { MODE_CONFIG, MODE_CONFIG_STAND_ALONE, toLoginPage } from '@/utils/login' import { ROLE_ADMIN } from '@/store/modules/user' NProgress.configure({ showSpinner: false }) // NProgress Configuration @@ -21,9 +21,10 @@ router.beforeEach(async(to, from, next) => { // determine whether the user has logged in // const bkTicket = getBkTicket() const bkUid = getBkUid() + const tempUid = getTempUid() const token = getToken() // const hasToken = token || bkTicket || bkUid - const hasToken = token || bkUid + const hasToken = token || bkUid || tempUid if (hasToken) { if (to.path === '/login') { // if is logged in, redirect to the home page @@ -71,6 +72,17 @@ router.beforeEach(async(to, from, next) => { next() } else { // other pages that do not have permission to access are redirected to the login page. + if (MODE_CONFIG !== MODE_CONFIG_STAND_ALONE) { + try { + const response = await fetch('/web/auth/api/user/info') + if (response.ok) { + const resJson = await response.json() + setTempUid(resJson.data.userId) + } + } catch (error) { + console.error(error) + } + } toLoginPage(to.path) NProgress.done() } diff --git a/src/frontend/devops-op/src/store/modules/user.js b/src/frontend/devops-op/src/store/modules/user.js index 7e75bd3182..1dab863646 100644 --- a/src/frontend/devops-op/src/store/modules/user.js +++ b/src/frontend/devops-op/src/store/modules/user.js @@ -1,5 +1,5 @@ import { login, userInfo, userInfoById } from '@/api/user' -import { getToken, setToken, removeToken, getBkUid } from '@/utils/auth' +import { getToken, setToken, removeToken, getBkUid, getTempUid } from '@/utils/auth' import { resetRouter } from '@/router' import { MODE_CONFIG, MODE_CONFIG_STAND_ALONE } from '@/utils/login' @@ -73,7 +73,7 @@ const actions = { // get user info async getInfo({ commit }) { - const uid = getBkUid() + const uid = getBkUid() || getTempUid() const user = (uid && MODE_CONFIG !== MODE_CONFIG_STAND_ALONE) ? uid : (await getUser()).user return new Promise((resolve, reject) => { userInfoById(user).then(response => { @@ -86,6 +86,7 @@ const actions = { } const avatar = '' + localStorage.setItem('userName', name) commit('SET_USER_ID', userId) commit('SET_NAME', name) commit('SET_ADMIN', admin) diff --git a/src/frontend/devops-op/src/utils/auth.js b/src/frontend/devops-op/src/utils/auth.js index 6b833646b0..eb961b91cc 100644 --- a/src/frontend/devops-op/src/utils/auth.js +++ b/src/frontend/devops-op/src/utils/auth.js @@ -4,6 +4,8 @@ export const TOKEN_KEY = 'bkrepo_ticket' export const BK_TICKET = 'bk_ticket' export const BK_UID = 'bk_uid' +const BK_TEMP_UID = 'bk_temp_uid' + export function getBkTicket() { return Cookies.get(BK_TICKET) } @@ -20,6 +22,14 @@ export function setToken(token) { return Cookies.set(TOKEN_KEY, token) } +export function setTempUid(uid) { + return Cookies.set(BK_TEMP_UID, uid) +} + +export function getTempUid() { + return Cookies.get(BK_TEMP_UID) +} + export function removeToken() { return Cookies.remove(TOKEN_KEY) } From 34eb08f39e3cec48c9c37448a4161d44d2790c08 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:55:26 +0800 Subject: [PATCH 07/11] =?UTF-8?q?feat:=20=E5=BD=93=E6=B2=A1=E6=9C=89?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E7=9A=84=E5=9C=BA=E6=99=AF=E4=B8=8B=EF=BC=8C?= =?UTF-8?q?=20=E5=90=8C=E6=AD=A5helm=E4=BB=93=E5=BA=93=E5=90=8E=E7=94=A8?= =?UTF-8?q?=E4=BA=8Eindex=E6=96=87=E4=BB=B6=E5=88=B7=E6=96=B0#2434?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 当没有事件的场景下， 同步helm仓库后用于index文件刷新#2434 * feat: 删除注释#2434 --- .../com/tencent/bkrepo/helm/api/HelmClient.kt | 9 ++ .../controller/service/HelmIndexController.kt | 22 ++- .../bkrepo/helm/utils/ObjectBuilderUtil.kt | 25 +++ .../task/refresh/HelmReplicationRefreshJob.kt | 151 ++++++++++++++++++ .../HelmReplicationRefreshJobProperties.kt | 37 +++++ 5 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/refresh/HelmReplicationRefreshJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/HelmReplicationRefreshJobProperties.kt diff --git a/src/backend/helm/api-helm/src/main/kotlin/com/tencent/bkrepo/helm/api/HelmClient.kt b/src/backend/helm/api-helm/src/main/kotlin/com/tencent/bkrepo/helm/api/HelmClient.kt index 73752713b7..73166e1959 100644 --- a/src/backend/helm/api-helm/src/main/kotlin/com/tencent/bkrepo/helm/api/HelmClient.kt +++ b/src/backend/helm/api-helm/src/main/kotlin/com/tencent/bkrepo/helm/api/HelmClient.kt @@ -66,6 +66,15 @@ interface HelmClient { @PathVariable repoName: String ): Response + @ApiOperation("当仓库有版本replication时，刷新index文件") + @PostMapping("/{projectId}/{repoName}/replication") + fun refreshIndexForReplication( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam packageName: String, + @RequestParam packageKey: String, + @RequestParam packageVersion: String, + ): Response @ApiOperation("删除仓库下的包版本") @DeleteMapping("version/delete/{projectId}/{repoName}") diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/service/HelmIndexController.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/service/HelmIndexController.kt index aa88bc5f27..d4d2d54fa7 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/service/HelmIndexController.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/controller/service/HelmIndexController.kt @@ -53,7 +53,7 @@ class HelmIndexController( private val remoteEventJobExecutor: RemoteEventJobExecutor, private val chartManipulationService: ChartManipulationService, private val fixToolService: FixToolService - ) : HelmClient { +) : HelmClient { /** * refresh index.yaml and package info for remote @@ -73,6 +73,26 @@ class HelmIndexController( return ResponseBuilder.success() } + /** + * refresh index.yaml for pacakge replication + */ + override fun refreshIndexForReplication( + projectId: String, repoName: String, + packageKey: String, packageName: String, + packageVersion: String, + ): Response { + val replicationEvent = ObjectBuilderUtil.buildPackageReplicationRequest( + projectId = projectId, + repoName = repoName, + packageName = packageName, + packageKey = packageKey, + packageVersion = packageVersion, + userId = SecurityUtils.getUserId() + ) + remoteEventJobExecutor.execute(replicationEvent) + return ResponseBuilder.success() + } + override fun deleteVersion( projectId: String, repoName: String, packageName: String, version: String diff --git a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/utils/ObjectBuilderUtil.kt b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/utils/ObjectBuilderUtil.kt index 9528af8977..8723383df4 100644 --- a/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/utils/ObjectBuilderUtil.kt +++ b/src/backend/helm/biz-helm/src/main/kotlin/com/tencent/bkrepo/helm/utils/ObjectBuilderUtil.kt @@ -35,6 +35,7 @@ import com.tencent.bkrepo.common.api.util.toYamlString import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.constant.ARTIFACT_INFO_KEY +import com.tencent.bkrepo.common.artifact.event.packages.VersionUpdatedEvent import com.tencent.bkrepo.common.artifact.event.repo.RepoCreatedEvent import com.tencent.bkrepo.common.artifact.event.repo.RepoRefreshedEvent import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -93,6 +94,29 @@ object ObjectBuilderUtil { ) } + /** + * 仓库有package replication事件 + */ + fun buildPackageReplicationRequest( + projectId: String, + repoName: String, + userId: String, + packageKey: String, + packageName: String, + packageVersion: String + ): VersionUpdatedEvent { + return VersionUpdatedEvent( + projectId = projectId, + repoName = repoName, + userId = userId, + packageType = PackageType.HELM.name, + packageKey = packageKey, + packageName = packageName, + packageVersion = packageVersion, + realIpAddress = null + ) + } + fun buildPackageUpdateRequest( projectId: String, repoName: String, @@ -184,6 +208,7 @@ object ObjectBuilderUtil { return PackageDownloadRecord(projectId, repoName, PackageKeys.ofHelm(name), version) } } + fun buildIndexYamlRequest(): ArtifactInfo { val artifactInfo = HttpContextHolder.getRequest().getAttribute(ARTIFACT_INFO_KEY) as ArtifactInfo return buildIndexYamlRequest(artifactInfo) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/refresh/HelmReplicationRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/refresh/HelmReplicationRefreshJob.kt new file mode 100644 index 0000000000..78059cc95d --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/refresh/HelmReplicationRefreshJob.kt @@ -0,0 +1,151 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.task.refresh + +import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.helm.api.HelmClient +import com.tencent.bkrepo.job.LAST_MODIFIED_DATE +import com.tencent.bkrepo.job.NAME +import com.tencent.bkrepo.job.TYPE +import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.config.properties.HelmReplicationRefreshJobProperties +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component +import java.time.Duration +import java.time.LocalDateTime +import kotlin.reflect.KClass + +/** + * 当没有事件的场景下， 同步helm仓库后用于index文件刷新 + */ +@Component +@EnableConfigurationProperties(HelmReplicationRefreshJobProperties::class) +class HelmReplicationRefreshJob( + private val properties: HelmReplicationRefreshJobProperties, + private val helmClient: HelmClient +) : DefaultContextMongoDbJob(properties) { + + + override fun entityClass(): KClass { + return Package::class + } + + override fun collectionNames(): List { + return listOf(COLLECTION_NAME) + } + + override fun buildQuery(): Query { + val fromDate = LocalDateTime.now().minusMinutes(1) + return Query( + Criteria.where(TYPE).`is`(RepositoryType.HELM.name) + .and(LAST_MODIFIED_DATE).gte(fromDate) + ) + } + + override fun getLockAtMostFor(): Duration = Duration.ofDays(1) + + override fun run(row: Package, collectionName: String, context: JobContext) { + with(row) { + try { + val query = Query(Criteria(PACKAGE_ID).isEqualTo(id).and(NAME).isEqualTo(latest)) + val versionData = mongoTemplate.findOne( + query, PackageVersionData::class.java, PACKAGE_VERSION_NAME + ) ?: return + if (!filterReplicationVersion(versionData)) return + logger.info( + "Preparing to send $row replication event " + + "for repo ${row.projectId}|${row.repoName}." + ) + helmClient.refreshIndexForReplication( + projectId = projectId, + repoName = repoName, + packageKey = key, + packageName = row.name, + packageVersion = versionData.name + ) + } catch (e: Exception) { + logger.warn( + "Failed to send $row replication event " + + "for repo ${row.projectId}|${row.repoName}, error: ${e.message}", e + ) + } + } + } + + override fun mapToEntity(row: Map): Package { + return Package(row) + } + + private fun filterReplicationVersion(versionData: PackageVersionData): Boolean { + with(versionData) { + if (metadata.isEmpty()) return false + val sourceType = metadata.firstOrNull { it[METADATA_KEY] == SOURCE_TYPE } + ?.get(METADATA_VALUE) as? String ?: return false + return sourceType == ArtifactChannel.REPLICATION.name + } + } + + data class Package( + val id: String, + var projectId: String, + var repoName: String, + var key: String, + var name: String, + var latest: String?, + ) { + constructor(map: Map) : this( + map[Package::id.name].toString(), + map[Package::projectId.name].toString(), + map[Package::repoName.name].toString(), + map[Package::key.name].toString(), + map[Package::name.name].toString(), + map[Package::latest.name]?.toString(), + ) + } + + data class PackageVersionData( + var name: String, + val metadata: List>, + ) + + companion object { + private val logger = LoggerHolder.jobLogger + private const val COLLECTION_NAME = "package" + private const val PACKAGE_VERSION_NAME = "package_version" + private const val PACKAGE_ID = "packageId" + private const val METADATA_KEY = "key" + private const val METADATA_VALUE = "value" + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/HelmReplicationRefreshJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/HelmReplicationRefreshJobProperties.kt new file mode 100644 index 0000000000..2a6ac8edd2 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/HelmReplicationRefreshJobProperties.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(value = "job.helm-replication-refresh") +data class HelmReplicationRefreshJobProperties( + override var enabled: Boolean = false, + override var fixedDelay: Long = 60 * 1000L, + override var initialDelay: Long = 60 * 1000L +): MongodbJobProperties() From 4d35c1019cbf24bf1da77d591ee2ceb90f81b675 Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Wed, 24 Jul 2024 16:09:45 +0800 Subject: [PATCH 08/11] =?UTF-8?q?feat:=20blockNode=E6=8A=BD=E5=8F=96?= =?UTF-8?q?=E8=87=B3=E5=85=AC=E5=85=B1=E6=A8=A1=E5=9D=97=20#2413?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/config/FsNodeDefaultImpl.kt | 14 -- .../common/common-metadata/build.gradle.kts | 30 ++++ .../metadata-api/build.gradle.kts | 30 ++++ .../common/metadata/constant/Constant.kt | 31 ++++ .../metadata-service/build.gradle.kts | 38 +++++ .../metadata/MetadataAutoConfiguration.kt | 40 +++++ .../common/metadata/dao/BlockNodeDao.kt | 35 ++++ .../metadata/dao/MetadataDaoConfiguration.kt | 37 +++++ .../common/metadata/model/TBlockNode.kt | 64 ++++++++ .../service/blocknode}/BlockNodeService.kt | 37 ++--- .../impl/AbstractBlockNodeService.kt | 150 ++++++++++++++++++ .../main/resources/META-INF/spring.factories | 29 ++++ .../bkrepo/fs/server/api/FsNodeClient.kt | 18 --- .../fs/boot-fs-server/build.gradle.kts | 1 + .../fs/server/config/RouteConfiguration.kt | 2 - .../server/handler/service/FsNodeHandler.kt | 17 +- .../bkrepo/fs/server/model/TBlockNode.kt | 64 -------- .../server/repository/BlockNodeRepository.kt | 35 ---- .../fs/server/service/BlockNodeServiceImpl.kt | 115 ++------------ .../fs/server/service/FileNodeService.kt | 1 + .../fs/server/service/FileOperationService.kt | 2 +- .../fs/server/storage/CoStorageManager.kt | 4 +- .../bkrepo/fs/service/BlockNodeServiceTest.kt | 12 +- .../biz-repository/build.gradle.kts | 1 + .../service/blocknode/BlockNodeServiceImpl.kt | 60 +++++++ .../service/node/impl/NodeBaseService.kt | 4 +- .../service/node/impl/NodeRestoreSupport.kt | 23 +-- .../service/node/impl/NodeServiceImpl.kt | 6 +- .../center/CommitEdgeCenterNodeServiceImpl.kt | 6 +- .../node/impl/edge/EdgeNodeBaseService.kt | 6 +- .../node/impl/edge/EdgeNodeServiceImpl.kt | 6 +- .../repository/service/ServiceBaseTest.kt | 2 - src/backend/settings.gradle.kts | 1 + 33 files changed, 611 insertions(+), 310 deletions(-) create mode 100644 src/backend/common/common-metadata/build.gradle.kts create mode 100644 src/backend/common/common-metadata/metadata-api/build.gradle.kts create mode 100644 src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt create mode 100644 src/backend/common/common-metadata/metadata-service/build.gradle.kts create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt rename src/backend/{fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service => common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode}/BlockNodeService.kt (52%) create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/resources/META-INF/spring.factories delete mode 100644 src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/TBlockNode.kt delete mode 100644 src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/repository/BlockNodeRepository.kt create mode 100644 src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt diff --git a/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt b/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt index b7f145c119..b405a2c13e 100644 --- a/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt +++ b/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt @@ -48,18 +48,4 @@ class FsNodeDefaultImpl : FsNodeClient { ): Response> { throw MethodNotAllowedException() } - - override fun restoreBlockResources( - projectId: String, - repoName: String, - fullPath: String, - nodeCreateDate: String, - nodeDeleteDate: String - ): Response { - throw MethodNotAllowedException() - } - - override fun deleteBlockResources(projectId: String, repoName: String, fullPath: String) { - throw MethodNotAllowedException() - } } diff --git a/src/backend/common/common-metadata/build.gradle.kts b/src/backend/common/common-metadata/build.gradle.kts new file mode 100644 index 0000000000..6971910f79 --- /dev/null +++ b/src/backend/common/common-metadata/build.gradle.kts @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ diff --git a/src/backend/common/common-metadata/metadata-api/build.gradle.kts b/src/backend/common/common-metadata/metadata-api/build.gradle.kts new file mode 100644 index 0000000000..6971910f79 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-api/build.gradle.kts @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ diff --git a/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt new file mode 100644 index 0000000000..bc1aa7340c --- /dev/null +++ b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.constant + +const val SHARDING_COUNT = 256 +const val ID = "_id" diff --git a/src/backend/common/common-metadata/metadata-service/build.gradle.kts b/src/backend/common/common-metadata/metadata-service/build.gradle.kts new file mode 100644 index 0000000000..5bc96239fb --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/build.gradle.kts @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +dependencies { + api(project(":common:common-artifact:artifact-api")) + api(project(":common:common-metadata:metadata-api")) + api(project(":common:common-mongo-reactive")) + api(project(":common:common-storage:storage-api")) + api(project(":repository:api-repository")) +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt new file mode 100644 index 0000000000..49ad3b2a54 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata + +import com.tencent.bkrepo.common.metadata.dao.MetadataDaoConfiguration +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Import + +@Configuration +@ConditionalOnWebApplication +@Import( + MetadataDaoConfiguration::class +) +class MetadataAutoConfiguration diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt new file mode 100644 index 0000000000..2e09ca3001 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.dao + +import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingMongoReactiveDao +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import org.springframework.stereotype.Repository + +@Repository +class BlockNodeDao : ShardingMongoReactiveDao() diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt new file mode 100644 index 0000000000..3b5c430747 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.dao + +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Import + +@Configuration +@Import( + BlockNodeDao::class +) +class MetadataDaoConfiguration diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt new file mode 100644 index 0000000000..a6d47edac1 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.model + +import com.tencent.bkrepo.common.metadata.constant.SHARDING_COUNT +import com.tencent.bkrepo.common.metadata.model.TBlockNode.Companion.BLOCK_IDX +import com.tencent.bkrepo.common.metadata.model.TBlockNode.Companion.BLOCK_IDX_DEF +import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingDocument +import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingKey +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import java.time.LocalDateTime + +/** + * 块节点 + * */ +@ShardingDocument("block_node") +@CompoundIndexes( + CompoundIndex(name = BLOCK_IDX, def = BLOCK_IDX_DEF) +) +data class TBlockNode( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + val nodeFullPath: String, + val startPos: Long, + var sha256: String, + val projectId: String, + @ShardingKey(count = SHARDING_COUNT) + val repoName: String, + val size: Long, + val endPos: Long = startPos + size - 1, + var deleted: LocalDateTime? = null +) { + companion object { + const val BLOCK_IDX = "start_pos_idx" + const val BLOCK_IDX_DEF = "{'projectId': 1, 'repoName': 1,'nodeFullPath':1, 'startPos': 1, 'isDeleted': 1}" + } +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt similarity index 52% rename from src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index aba1d01b58..faf875679c 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -1,40 +1,37 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * - * A copy of the MIT License is included in this file. + * A copy of the MIT License is included in this file. * * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.fs.server.service +package com.tencent.bkrepo.common.metadata.service.blocknode import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.fs.server.model.TBlockNode import java.time.LocalDateTime -/** - * 块服务 - * */ interface BlockNodeService { /** * 查询出范围内的分块 diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt new file mode 100644 index 0000000000..6e67b1c68c --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt @@ -0,0 +1,150 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.service.blocknode.impl + +import com.tencent.bkrepo.common.api.util.EscapeUtils +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.constant.ID +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import org.slf4j.LoggerFactory +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.gt +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.lt +import org.springframework.data.mongodb.core.query.where +import java.time.LocalDateTime + +abstract class AbstractBlockNodeService( + private val blockNodeDao: BlockNodeDao +) : BlockNodeService { + + override suspend fun createBlock( + blockNode: TBlockNode, + storageCredentials: StorageCredentials? + ): TBlockNode { + with(blockNode) { + val bn = blockNodeDao.save(blockNode) + incFileRef(bn.sha256, storageCredentials?.key) + logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.") + return bn + } + } + + override suspend fun listBlocks( + range: Range, + projectId: String, + repoName: String, + fullPath: String, + createdDate: String + ): List { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + .and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) + .norOperator( + TBlockNode::startPos.gt(range.end), + TBlockNode::endPos.lt(range.start) + ) + val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) + return blockNodeDao.find(query) + } + + override suspend fun deleteBlocks( + projectId: String, + repoName: String, + fullPath: String + ) { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + val update = Update().set(TBlockNode::deleted.name, LocalDateTime.now()) + blockNodeDao.updateMulti(Query(criteria), update) + logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") + } + + override suspend fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { + val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) + if (nodeDetail.folder) { + val criteria = where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + val blocks = blockNodeDao.find(Query(criteria)) + blocks.forEach { + val update = Update().set(TBlockNode::nodeFullPath.name, it.nodeFullPath.replace(fullPath, dstFullPath)) + val query = Query(Criteria.where(ID).isEqualTo(it.id).and(TBlockNode::repoName).isEqualTo(repoName)) + blockNodeDao.updateMulti(query, update) + } + } else { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + val update = Update().set(TBlockNode::nodeFullPath.name, dstFullPath) + blockNodeDao.updateMulti(Query(criteria), update) + } + logger.info("Move node[$projectId/$repoName$fullPath] to node[$projectId/$repoName$dstFullPath] success.") + } + + + + override suspend fun restoreBlocks( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ) { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::createdDate).gt(nodeCreateDate).lt(nodeDeleteDate) + val update = Update().set(TBlockNode::deleted.name, null) + val result = blockNodeDao.updateMulti(Query(criteria), update) + logger.info("Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + + "between $nodeCreateDate and $nodeDeleteDate success.") + } + + abstract suspend fun incFileRef(sha256: String, credentialsKey: String?) + + abstract suspend fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail + + companion object { + private val logger = LoggerFactory.getLogger(AbstractBlockNodeService::class.java) + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/resources/META-INF/spring.factories b/src/backend/common/common-metadata/metadata-service/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..4a292034f6 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/resources/META-INF/spring.factories @@ -0,0 +1,29 @@ +# +# Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. +# +# Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. +# +# BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. +# +# A copy of the MIT License is included in this file. +# +# +# Terms of the MIT License: +# --------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bkrepo.common.metadata.MetadataAutoConfiguration diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt index 9853d201bf..76fd96a2cf 100644 --- a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt +++ b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt @@ -31,10 +31,8 @@ import com.tencent.bkrepo.common.api.constant.FS_SERVER_SERVICE_NAME import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.storage.pojo.RegionResource import org.springframework.cloud.openfeign.FeignClient -import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam @@ -50,20 +48,4 @@ interface FsNodeClient { @RequestParam startPos: Long, @RequestParam endPos: Long ): Response> - - @PostMapping("/restore/{projectId}/{repoName}") - fun restoreBlockResources( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam fullPath: String, - @RequestParam nodeCreateDate: String, - @RequestParam nodeDeleteDate: String, - ): Response - - @DeleteMapping("/delete/{projectId}/{repoName}") - fun deleteBlockResources( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam fullPath: String - ) } diff --git a/src/backend/fs/boot-fs-server/build.gradle.kts b/src/backend/fs/boot-fs-server/build.gradle.kts index 143faa5d33..a990c5da36 100644 --- a/src/backend/fs/boot-fs-server/build.gradle.kts +++ b/src/backend/fs/boot-fs-server/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { api(project(":common:common-stream")) api(project(":fs:api-fs-server")) implementation("com.github.ben-manes.caffeine:caffeine:2.9.3") + implementation(project(":common:common-metadata:metadata-service")) testImplementation("org.mockito.kotlin:mockito-kotlin") testImplementation("de.flapdoodle.embed:de.flapdoodle.embed.mongo") diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt index b6968a9c2f..2457a1685c 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt @@ -86,8 +86,6 @@ class RouteConfiguration( "/service/block".nest { GET("/list$DEFAULT_MAPPING_URI", fsNodeHandler::listBlocks) - POST("/restore$DEFAULT_MAPPING_URI", fsNodeHandler::restoreBlock) - DELETE("/delete$DEFAULT_MAPPING_URI", fsNodeHandler::deleteBlock) } "/node".nest { diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt index 829a5fc5f3..c26f7eeb31 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt @@ -28,11 +28,11 @@ package com.tencent.bkrepo.fs.server.handler.service import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.fs.server.api.RRepositoryClient import com.tencent.bkrepo.fs.server.request.service.DeleteBlocksRequest import com.tencent.bkrepo.fs.server.request.service.ListBlocksRequest import com.tencent.bkrepo.fs.server.request.service.RestoreBlocksRequest -import com.tencent.bkrepo.fs.server.service.BlockNodeService import com.tencent.bkrepo.fs.server.service.FileNodeService import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder import kotlinx.coroutines.reactor.awaitSingle @@ -57,19 +57,4 @@ class FsNodeHandler( return ReactiveResponseBuilder.success(fileNodeService.info(nodeDetail, range)) } } - - suspend fun restoreBlock(request: ServerRequest): ServerResponse { - with(RestoreBlocksRequest(request)) { - blockNodeService.restoreBlocks(projectId, repoName, fullPath, nodeCreateDate, nodeDeleteDate) - return ReactiveResponseBuilder.success() - } - } - - suspend fun deleteBlock(request: ServerRequest): ServerResponse { - with(DeleteBlocksRequest(request)) { - blockNodeService.deleteBlocks(projectId, repoName, fullPath) - return ReactiveResponseBuilder.success() - } - - } } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/TBlockNode.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/TBlockNode.kt deleted file mode 100644 index bafc4b9284..0000000000 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/TBlockNode.kt +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.fs.server.model - -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingDocument -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingKey -import com.tencent.bkrepo.fs.server.model.TBlockNode.Companion.BLOCK_IDX -import com.tencent.bkrepo.fs.server.model.TBlockNode.Companion.BLOCK_IDX_DEF -import com.tencent.bkrepo.repository.constant.SHARDING_COUNT -import org.springframework.data.mongodb.core.index.CompoundIndex -import org.springframework.data.mongodb.core.index.CompoundIndexes -import java.time.LocalDateTime - -/** - * 块节点 - * */ -@ShardingDocument("block_node") -@CompoundIndexes( - CompoundIndex(name = BLOCK_IDX, def = BLOCK_IDX_DEF) -) -data class TBlockNode( - var id: String? = null, - var createdBy: String, - var createdDate: LocalDateTime, - val nodeFullPath: String, - val startPos: Long, - var sha256: String, - val projectId: String, - @ShardingKey(count = SHARDING_COUNT) - val repoName: String, - val size: Long, - val endPos: Long = startPos + size - 1, - var deleted: LocalDateTime? = null -) { - companion object { - const val BLOCK_IDX = "start_pos_idx" - const val BLOCK_IDX_DEF = "{'projectId': 1, 'repoName': 1,'nodeFullPath':1, 'startPos': 1, 'isDeleted': 1}" - } -} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/repository/BlockNodeRepository.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/repository/BlockNodeRepository.kt deleted file mode 100644 index 2e6d607082..0000000000 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/repository/BlockNodeRepository.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.fs.server.repository - -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingMongoReactiveDao -import com.tencent.bkrepo.fs.server.model.TBlockNode -import org.springframework.stereotype.Repository - -@Repository -class BlockNodeRepository : ShardingMongoReactiveDao() diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt index 51fd09ee1b..fe24eb906a 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt @@ -27,124 +27,27 @@ package com.tencent.bkrepo.fs.server.service -import com.tencent.bkrepo.common.api.util.EscapeUtils import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.service.blocknode.impl.AbstractBlockNodeService import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.fs.server.constant.ID -import com.tencent.bkrepo.fs.server.model.TBlockNode -import com.tencent.bkrepo.fs.server.repository.BlockNodeRepository +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import kotlinx.coroutines.reactor.awaitSingle -import org.slf4j.LoggerFactory -import org.springframework.data.domain.Sort -import org.springframework.data.mongodb.core.query.Criteria -import org.springframework.data.mongodb.core.query.Query -import org.springframework.data.mongodb.core.query.Update -import org.springframework.data.mongodb.core.query.and -import org.springframework.data.mongodb.core.query.gt -import org.springframework.data.mongodb.core.query.isEqualTo -import org.springframework.data.mongodb.core.query.lt -import org.springframework.data.mongodb.core.query.where -import java.time.LocalDateTime /** * 文件块服务 * */ class BlockNodeServiceImpl( - private val blockNodeRepository: BlockNodeRepository, + blockNodeDao: BlockNodeDao, private val rRepositoryClient: RRepositoryClient -) : BlockNodeService { +) : AbstractBlockNodeService(blockNodeDao) { - override suspend fun createBlock( - blockNode: TBlockNode, - storageCredentials: StorageCredentials? - ): TBlockNode { - with(blockNode) { - val bn = blockNodeRepository.save(blockNode) - rRepositoryClient.increment(blockNode.sha256, storageCredentials?.key).awaitSingle() - logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.") - return bn - } + override suspend fun incFileRef(sha256: String, credentialsKey: String?) { + rRepositoryClient.increment(sha256, credentialsKey).awaitSingle() } - override suspend fun listBlocks( - range: Range, - projectId: String, - repoName: String, - fullPath: String, - createdDate: String - ): List { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - .and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) - .norOperator( - TBlockNode::startPos.gt(range.end), - TBlockNode::endPos.lt(range.start) - ) - val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) - return blockNodeRepository.find(query) - } - - override suspend fun deleteBlocks( - projectId: String, - repoName: String, - fullPath: String - ) { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val update = Update().set(TBlockNode::deleted.name, LocalDateTime.now()) - blockNodeRepository.updateMulti(Query(criteria), update) - logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") - } - - override suspend fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { - val nodeDetail = rRepositoryClient.getNodeDetail(projectId, repoName, dstFullPath).awaitSingle().data + override suspend fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail { + return rRepositoryClient.getNodeDetail(projectId, repoName, dstFullPath).awaitSingle().data ?: throw NodeNotFoundException(dstFullPath) - if (nodeDetail.folder) { - val criteria = where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val blocks = blockNodeRepository.find(Query(criteria)) - blocks.forEach { - val update = Update().set(TBlockNode::nodeFullPath.name, it.nodeFullPath.replace(fullPath, dstFullPath)) - val query = Query(Criteria.where(ID).isEqualTo(it.id).and(TBlockNode::repoName).isEqualTo(repoName)) - blockNodeRepository.updateMulti(query, update) - } - } else { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val update = Update().set(TBlockNode::nodeFullPath.name, dstFullPath) - blockNodeRepository.updateMulti(Query(criteria), update) - } - logger.info("Move node[$projectId/$repoName$fullPath] to node[$projectId/$repoName$dstFullPath] success.") - } - - override suspend fun restoreBlocks( - projectId: String, - repoName: String, - fullPath: String, - nodeCreateDate: LocalDateTime, - nodeDeleteDate: LocalDateTime - ) { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::createdDate).gt(nodeCreateDate).lt(nodeDeleteDate) - val update = Update().set(TBlockNode::deleted.name, null) - val result = blockNodeRepository.updateMulti(Query(criteria), update) - logger.info("Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + - "between $nodeCreateDate and $nodeDeleteDate success.") - } - - companion object { - private val logger = LoggerFactory.getLogger(BlockNodeServiceImpl::class.java) } } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt index bbe381c47c..121139edd3 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt @@ -30,6 +30,7 @@ package com.tencent.bkrepo.fs.server.service import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt index 83bbcf5470..75b1e48e21 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt @@ -33,7 +33,7 @@ import com.tencent.bkrepo.fs.server.api.RRepositoryClient import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder import com.tencent.bkrepo.fs.server.model.NodeAttribute -import com.tencent.bkrepo.fs.server.model.TBlockNode +import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.fs.server.request.BlockRequest import com.tencent.bkrepo.fs.server.request.FlushRequest import com.tencent.bkrepo.fs.server.storage.CoStorageManager diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt index 8fe79860e7..e0ceb4abdb 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt @@ -30,12 +30,12 @@ package com.tencent.bkrepo.fs.server.storage import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.fs.server.RepositoryCache -import com.tencent.bkrepo.fs.server.model.TBlockNode -import com.tencent.bkrepo.fs.server.service.BlockNodeService import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext diff --git a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt index 8ff6833fdf..e27bdcfec3 100644 --- a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt +++ b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt @@ -6,9 +6,9 @@ import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.fs.server.model.TBlockNode -import com.tencent.bkrepo.fs.server.repository.BlockNodeRepository -import com.tencent.bkrepo.fs.server.service.BlockNodeService +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.com.bkrepo.fs.UT_PROJECT_ID @@ -36,7 +36,7 @@ import reactor.core.publisher.Mono import java.time.LocalDateTime @DataMongoTest -@Import(BlockNodeRepository::class) +@Import(BlockNodeDao::class) @SpringBootConfiguration @EnableAutoConfiguration @TestPropertySource(locations = ["classpath:bootstrap-ut.properties"]) @@ -49,13 +49,13 @@ class BlockNodeServiceTest { lateinit var blockNodeService: BlockNodeService @Autowired - lateinit var blockNodeRepository: BlockNodeRepository + lateinit var blockNodeDao: BlockNodeDao private val storageCredentials = FileSystemCredentials() @BeforeEach fun beforeEach() { val criteria = where(TBlockNode::repoName).isEqualTo(UT_REPO_NAME) - runBlocking { blockNodeRepository.remove(Query(criteria)) } + runBlocking { blockNodeDao.remove(Query(criteria)) } } @DisplayName("测试创建块") diff --git a/src/backend/repository/biz-repository/build.gradle.kts b/src/backend/repository/biz-repository/build.gradle.kts index c3ac8bd752..e992c620ab 100644 --- a/src/backend/repository/biz-repository/build.gradle.kts +++ b/src/backend/repository/biz-repository/build.gradle.kts @@ -42,4 +42,5 @@ dependencies { testImplementation("org.mockito.kotlin:mockito-kotlin") testImplementation("io.mockk:mockk") implementation("com.tencent.bk.sdk:bk-notice-java-sdk:${Versions.Notice}") + implementation(project(":common:common-metadata:metadata-service")) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt new file mode 100644 index 0000000000..2ebaf6fc90 --- /dev/null +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.repository.service.blocknode + +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.service.blocknode.impl.AbstractBlockNodeService +import com.tencent.bkrepo.repository.dao.NodeDao +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.springframework.stereotype.Service + +@Service +class BlockNodeServiceImpl( + blockNodeDao: BlockNodeDao, + private val fileReferenceService: FileReferenceService, + private val nodeDao: NodeDao +) : AbstractBlockNodeService(blockNodeDao) { + + override suspend fun incFileRef(sha256: String, credentialsKey: String?) { + withContext(Dispatchers.IO) { + fileReferenceService.increment(sha256, credentialsKey) + } + } + + override suspend fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail { + return withContext(Dispatchers.IO) { + val node = nodeDao.findNode(projectId, repoName, dstFullPath) ?: throw NodeNotFoundException(dstFullPath) + NodeBaseService.convertToDetail(node)!! + } + } +} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt index 36d9573364..5674e117fa 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt @@ -42,6 +42,7 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.query.model.Sort import com.tencent.bkrepo.common.security.manager.PermissionManager @@ -51,7 +52,6 @@ import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publi import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.constant.BinderType import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.fs.server.constant.FAKE_MD5 import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.config.RepositoryProperties @@ -105,7 +105,7 @@ abstract class NodeBaseService( open val servicePermissionClient: ServicePermissionClient, open val routerControllerClient: RouterControllerClient, open val routerControllerProperties: RouterControllerProperties, - open val fsNodeClient: FsNodeClient + open val blockNodeService: BlockNodeService ) : NodeService { @Autowired diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt index 1ff2302842..1694091b3d 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt @@ -35,8 +35,8 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils.isRoot +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.repository.dao.NodeDao @@ -58,6 +58,7 @@ import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeListQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeRestoreUpdate +import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory import org.springframework.dao.DuplicateKeyException import org.springframework.data.mongodb.core.FindAndModifyOptions @@ -73,7 +74,7 @@ open class NodeRestoreSupport( ) : NodeRestoreOperation { val nodeDao: NodeDao = nodeBaseService.nodeDao - val fsNodeClient: FsNodeClient = nodeBaseService.fsNodeClient + val blockNodeService: BlockNodeService = nodeBaseService.blockNodeService override fun getDeletedNodeDetail(artifact: ArtifactInfo): List { with(artifact) { @@ -178,13 +179,15 @@ open class NodeRestoreSupport( val deletedNode = nodeDao.findAndModify(query, nodeRestoreUpdate(), option, TNode::class.java) if (deletedNode?.sha256 == FAKE_SHA256 || deletedNode?.metadata?.find { it.key == FS_ATTR_KEY } != null) { try { - fsNodeClient.restoreBlockResources( - projectId = projectId, - repoName = repoName, - fullPath = fullPath, - nodeCreateDate = deletedNode.createdDate.toString(), - nodeDeleteDate = deletedNode.deleted!!.toString() - ) + runBlocking { + blockNodeService.restoreBlocks( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + nodeCreateDate = deletedNode.createdDate, + nodeDeleteDate = deletedNode.deleted!! + ) + } } catch (e: Exception) { logger.error("restore block resources failed: $projectId/$repoName/$fullPath", e) nodeDao.save(deletedNode) @@ -202,7 +205,7 @@ open class NodeRestoreSupport( with(node) { val query = nodeQuery(projectId, repoName, fullPath) if (node.sha256 == FAKE_SHA256 || node.metadata?.find { it.key == FS_ATTR_KEY } != null) { - fsNodeClient.deleteBlockResources(projectId, repoName, fullPath) + runBlocking { blockNodeService.deleteBlocks(projectId, repoName, fullPath) } } nodeDao.updateFirst(query, NodeQueryHelper.nodeDeleteUpdate(userId)) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt index 98a7bd8351..86bc0a7987 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt @@ -31,10 +31,10 @@ import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.dao.RepositoryDao @@ -76,7 +76,7 @@ class NodeServiceImpl( override val servicePermissionClient: ServicePermissionClient, override val routerControllerClient: RouterControllerClient, override val routerControllerProperties: RouterControllerProperties, - override val fsNodeClient: FsNodeClient, + override val blockNodeService: BlockNodeService, private val archiveClient: ArchiveClient, ) : NodeBaseService( nodeDao, @@ -90,7 +90,7 @@ class NodeServiceImpl( servicePermissionClient, routerControllerClient, routerControllerProperties, - fsNodeClient + blockNodeService ) { override fun computeSize( diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt index 1702ae0a57..3ded79ab00 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt @@ -35,12 +35,12 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties import com.tencent.bkrepo.common.artifact.util.ClusterUtils +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.dao.RepositoryDao @@ -84,7 +84,7 @@ class CommitEdgeCenterNodeServiceImpl( override val servicePermissionClient: ServicePermissionClient, override val routerControllerClient: RouterControllerClient, override val routerControllerProperties: RouterControllerProperties, - override val fsNodeClient: FsNodeClient, + override val blockNodeService: BlockNodeService, val clusterProperties: ClusterProperties, val archiveClient: ArchiveClient, ) : NodeServiceImpl( @@ -99,7 +99,7 @@ class CommitEdgeCenterNodeServiceImpl( servicePermissionClient, routerControllerClient, routerControllerProperties, - fsNodeClient, + blockNodeService, archiveClient ) { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt index 0f7084ed58..bcc9388a07 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt @@ -30,11 +30,11 @@ package com.tencent.bkrepo.repository.service.node.impl.edge import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties import com.tencent.bkrepo.common.artifact.util.ClusterUtils.reportMetadataToCenter +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.feign.FeignClientFactory import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.api.cluster.ClusterNodeClient import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.NodeDao @@ -62,7 +62,7 @@ abstract class EdgeNodeBaseService( override val routerControllerClient: RouterControllerClient, override val servicePermissionClient: ServicePermissionClient, override val routerControllerProperties: RouterControllerProperties, - override val fsNodeClient: FsNodeClient, + override val blockNodeService: BlockNodeService, open val clusterProperties: ClusterProperties ) : NodeBaseService( nodeDao, @@ -76,7 +76,7 @@ abstract class EdgeNodeBaseService( servicePermissionClient, routerControllerClient, routerControllerProperties, - fsNodeClient + blockNodeService ) { val centerNodeClient: ClusterNodeClient by lazy { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt index 493476f95c..cf723ba665 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -34,11 +34,11 @@ import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties import com.tencent.bkrepo.common.artifact.util.ClusterUtils.ignoreException import com.tencent.bkrepo.common.artifact.util.ClusterUtils.nodeLevelNotFoundError import com.tencent.bkrepo.common.artifact.util.ClusterUtils.repoLevelNotFoundError +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.dao.RepositoryDao @@ -89,7 +89,7 @@ class EdgeNodeServiceImpl( override val clusterProperties: ClusterProperties, override val routerControllerClient: RouterControllerClient, override val routerControllerProperties: RouterControllerProperties, - override val fsNodeClient: FsNodeClient, + override val blockNodeService: BlockNodeService, val archiveClient: ArchiveClient, ) : EdgeNodeBaseService( nodeDao, @@ -103,7 +103,7 @@ class EdgeNodeServiceImpl( routerControllerClient, servicePermissionClient, routerControllerProperties, - fsNodeClient, + blockNodeService, clusterProperties, ) { override fun computeSize( diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt index 614c400906..d7022a6902 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt @@ -177,8 +177,6 @@ open class ServiceBaseTest { whenever(messageSupplier.delegateToSupplier(any(), anyOrNull(), anyString(), anyOrNull(), any())) .then {} whenever(resourcePermissionListener.handle(any())).then {} - whenever(fsNodeClient.restoreBlockResources(anyString(), anyString(), anyString(), anyString(), anyString())) - .then {} } fun initRepoForUnitTest( diff --git a/src/backend/settings.gradle.kts b/src/backend/settings.gradle.kts index 25925627e1..4d0fae6d4f 100644 --- a/src/backend/settings.gradle.kts +++ b/src/backend/settings.gradle.kts @@ -91,3 +91,4 @@ includeAll(":archive") includeAll(":s3") includeAll(":router-controller") includeAll(":media") +includeAll(":common:common-metadata") From 3687776bb6ba4d5ba1ce357f5adf9356178033eb Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Thu, 1 Aug 2024 16:09:12 +0800 Subject: [PATCH 09/11] =?UTF-8?q?feat:=20blockNode=E6=8A=BD=E5=8F=96?= =?UTF-8?q?=E8=87=B3=E5=85=AC=E5=85=B1=E6=A8=A1=E5=9D=97=20#2413?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../metadata-service/build.gradle.kts | 4 +- .../metadata/condition/ReactiveCondition.kt | 16 +++ .../metadata/condition/SyncCondition.kt | 16 +++ .../common/metadata/dao/BlockNodeDao.kt | 34 +---- .../metadata/dao/MetadataDaoConfiguration.kt | 6 +- .../common/metadata/dao/RBlockNodeDao.kt | 38 ++++++ .../service/blocknode/BlockNodeService.kt | 17 ++- .../service/blocknode/RBlockNodeService.kt | 61 +++++++++ .../impl/AbstractBlockNodeService.kt | 17 +-- .../impl/RAbstractBlockNodeService.kt | 120 ++++++++++++++++++ .../bkrepo/fs/server/handler/ClientHandler.kt | 4 +- .../server/handler/service/FsNodeHandler.kt | 6 +- .../fs/server/service/BlockNodeServiceImpl.kt | 8 +- .../fs/server/service/FileNodeService.kt | 10 +- .../fs/server/storage/CoStorageManager.kt | 4 +- .../bkrepo/fs/service/BlockNodeServiceTest.kt | 12 +- .../service/blocknode/BlockNodeServiceImpl.kt | 16 +-- .../service/node/impl/NodeRestoreSupport.kt | 19 ++- 18 files changed, 311 insertions(+), 97 deletions(-) create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt diff --git a/src/backend/common/common-metadata/metadata-service/build.gradle.kts b/src/backend/common/common-metadata/metadata-service/build.gradle.kts index 5bc96239fb..8fba1d187c 100644 --- a/src/backend/common/common-metadata/metadata-service/build.gradle.kts +++ b/src/backend/common/common-metadata/metadata-service/build.gradle.kts @@ -32,7 +32,9 @@ dependencies { api(project(":common:common-artifact:artifact-api")) api(project(":common:common-metadata:metadata-api")) - api(project(":common:common-mongo-reactive")) api(project(":common:common-storage:storage-api")) api(project(":repository:api-repository")) + + compileOnly(project(":common:common-mongo-reactive")) + compileOnly(project(":common:common-mongo")) } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt new file mode 100644 index 0000000000..d2b55647f1 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt @@ -0,0 +1,16 @@ +package com.tencent.bkrepo.common.metadata.condition + +import org.springframework.context.annotation.Condition +import org.springframework.context.annotation.ConditionContext +import org.springframework.core.type.AnnotatedTypeMetadata + +class ReactiveCondition : Condition { + override fun matches(context: ConditionContext, metadata: AnnotatedTypeMetadata): Boolean { + try { + context.classLoader?.loadClass("com.mongodb.reactivestreams.client.MongoClient") + return true + } catch (e: ClassNotFoundException) { + return false + } + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt new file mode 100644 index 0000000000..6b17441a28 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt @@ -0,0 +1,16 @@ +package com.tencent.bkrepo.common.metadata.condition + +import org.springframework.context.annotation.Condition +import org.springframework.context.annotation.ConditionContext +import org.springframework.core.type.AnnotatedTypeMetadata + +class SyncCondition : Condition { + override fun matches(context: ConditionContext, metadata: AnnotatedTypeMetadata): Boolean { + try { + context.classLoader?.loadClass("com.mongodb.client.MongoClient") + return true + } catch (e: ClassNotFoundException) { + return false + } + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt index 2e09ca3001..779e428c13 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt @@ -1,35 +1,11 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - package com.tencent.bkrepo.common.metadata.dao -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingMongoReactiveDao +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.mongo.dao.sharding.HashShardingMongoDao +import org.springframework.context.annotation.Conditional import org.springframework.stereotype.Repository @Repository -class BlockNodeDao : ShardingMongoReactiveDao() +@Conditional(SyncCondition::class) +class BlockNodeDao : HashShardingMongoDao() diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt index 3b5c430747..5b41522a3f 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt @@ -27,11 +27,9 @@ package com.tencent.bkrepo.common.metadata.dao +import org.springframework.context.annotation.ComponentScan import org.springframework.context.annotation.Configuration -import org.springframework.context.annotation.Import @Configuration -@Import( - BlockNodeDao::class -) +@ComponentScan(basePackages = ["com.tencent.bkrepo.common.metadata.dao"]) class MetadataDaoConfiguration diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt new file mode 100644 index 0000000000..7ccb332148 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.dao + +import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingMongoReactiveDao +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Repository + +@Repository +@Conditional(ReactiveCondition::class) +class RBlockNodeDao : ShardingMongoReactiveDao() diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index faf875679c..16e48f2ad8 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -33,10 +33,11 @@ import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import java.time.LocalDateTime interface BlockNodeService { + /** * 查询出范围内的分块 * */ - suspend fun listBlocks( + fun listBlocks( range: Range, projectId: String, repoName: String, @@ -47,7 +48,7 @@ interface BlockNodeService { /** * 创建分块 * */ - suspend fun createBlock( + fun createBlock( blockNode: TBlockNode, storageCredentials: StorageCredentials? ): TBlockNode @@ -59,20 +60,26 @@ interface BlockNodeService { * @param repoName 仓库名 * @param fullPath 文件路径 * */ - suspend fun deleteBlocks( + fun deleteBlocks( projectId: String, repoName: String, fullPath: String ) - suspend fun moveBlocks( + /** + * 移动文件对应分块 + */ + fun moveBlocks( projectId: String, repoName: String, fullPath: String, dstFullPath: String ) - suspend fun restoreBlocks( + /** + * 恢复文件对应分块 + */ + fun restoreBlocks( projectId: String, repoName: String, fullPath: String, diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt new file mode 100644 index 0000000000..f0ccf1015a --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt @@ -0,0 +1,61 @@ +package com.tencent.bkrepo.common.metadata.service.blocknode + +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import java.time.LocalDateTime + +interface RBlockNodeService { + /** + * 查询出范围内的分块 + * */ + suspend fun listBlocks( + range: Range, + projectId: String, + repoName: String, + fullPath: String, + createdDate: String + ): List + + /** + * 创建分块 + * */ + suspend fun createBlock( + blockNode: TBlockNode, + storageCredentials: StorageCredentials? + ): TBlockNode + + /** + * 删除旧分块，即删除非指定的nodeCurrentSha256的分块。 + * 如果未指定nodeCurrentSha256，则删除节点所有分块 + * @param projectId 项目id + * @param repoName 仓库名 + * @param fullPath 文件路径 + * */ + suspend fun deleteBlocks( + projectId: String, + repoName: String, + fullPath: String + ) + + /** + * 移动文件对应分块 + */ + suspend fun moveBlocks( + projectId: String, + repoName: String, + fullPath: String, + dstFullPath: String + ) + + /** + * 恢复文件对应分块 + */ + suspend fun restoreBlocks( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ) +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt index 6e67b1c68c..ec143adb29 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt @@ -51,10 +51,7 @@ abstract class AbstractBlockNodeService( private val blockNodeDao: BlockNodeDao ) : BlockNodeService { - override suspend fun createBlock( - blockNode: TBlockNode, - storageCredentials: StorageCredentials? - ): TBlockNode { + override fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode { with(blockNode) { val bn = blockNodeDao.save(blockNode) incFileRef(bn.sha256, storageCredentials?.key) @@ -63,7 +60,7 @@ abstract class AbstractBlockNodeService( } } - override suspend fun listBlocks( + override fun listBlocks( range: Range, projectId: String, repoName: String, @@ -83,7 +80,7 @@ abstract class AbstractBlockNodeService( return blockNodeDao.find(query) } - override suspend fun deleteBlocks( + override fun deleteBlocks( projectId: String, repoName: String, fullPath: String @@ -97,7 +94,7 @@ abstract class AbstractBlockNodeService( logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") } - override suspend fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { + override fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) if (nodeDetail.folder) { val criteria = where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") @@ -123,7 +120,7 @@ abstract class AbstractBlockNodeService( - override suspend fun restoreBlocks( + override fun restoreBlocks( projectId: String, repoName: String, fullPath: String, @@ -140,9 +137,9 @@ abstract class AbstractBlockNodeService( "between $nodeCreateDate and $nodeDeleteDate success.") } - abstract suspend fun incFileRef(sha256: String, credentialsKey: String?) + abstract fun incFileRef(sha256: String, credentialsKey: String?) - abstract suspend fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail + abstract fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail companion object { private val logger = LoggerFactory.getLogger(AbstractBlockNodeService::class.java) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt new file mode 100644 index 0000000000..6fb28d1660 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt @@ -0,0 +1,120 @@ +package com.tencent.bkrepo.common.metadata.service.blocknode.impl + +import com.tencent.bkrepo.common.api.util.EscapeUtils +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.constant.ID +import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import org.slf4j.LoggerFactory +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.gt +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.lt +import org.springframework.data.mongodb.core.query.where +import java.time.LocalDateTime + +abstract class RAbstractBlockNodeService( + private val rBlockNodeDao: RBlockNodeDao +) : RBlockNodeService { + + override suspend fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode { + with(blockNode) { + val bn = rBlockNodeDao.save(blockNode) + incFileRef(bn.sha256, storageCredentials?.key) + logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.") + return bn + } + } + + override suspend fun listBlocks( + range: Range, + projectId: String, + repoName: String, + fullPath: String, + createdDate: String + ): List { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + .and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) + .norOperator( + TBlockNode::startPos.gt(range.end), + TBlockNode::endPos.lt(range.start) + ) + val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) + return rBlockNodeDao.find(query) + } + + override suspend fun deleteBlocks( + projectId: String, + repoName: String, + fullPath: String + ) { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + val update = Update().set(TBlockNode::deleted.name, LocalDateTime.now()) + rBlockNodeDao.updateMulti(Query(criteria), update) + logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") + } + + override suspend fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { + val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) + if (nodeDetail.folder) { + val criteria = where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + val blocks = rBlockNodeDao.find(Query(criteria)) + blocks.forEach { + val update = Update().set(TBlockNode::nodeFullPath.name, it.nodeFullPath.replace(fullPath, dstFullPath)) + val query = Query(Criteria.where(ID).isEqualTo(it.id).and(TBlockNode::repoName).isEqualTo(repoName)) + rBlockNodeDao.updateMulti(query, update) + } + } else { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + val update = Update().set(TBlockNode::nodeFullPath.name, dstFullPath) + rBlockNodeDao.updateMulti(Query(criteria), update) + } + logger.info("Move node[$projectId/$repoName$fullPath] to node[$projectId/$repoName$dstFullPath] success.") + } + + + + override suspend fun restoreBlocks( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ) { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::createdDate).gt(nodeCreateDate).lt(nodeDeleteDate) + val update = Update().set(TBlockNode::deleted.name, null) + val result = rBlockNodeDao.updateMulti(Query(criteria), update) + logger.info("Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + + "between $nodeCreateDate and $nodeDeleteDate success.") + } + + abstract suspend fun incFileRef(sha256: String, credentialsKey: String?) + + abstract suspend fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail + + companion object { + private val logger = LoggerFactory.getLogger(RAbstractBlockNodeService::class.java) + } +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/ClientHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/ClientHandler.kt index bb87d8be8c..e5bddf9533 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/ClientHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/ClientHandler.kt @@ -29,9 +29,9 @@ package com.tencent.bkrepo.fs.server.handler import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE -import com.tencent.bkrepo.fs.server.request.ClientCreateRequest import com.tencent.bkrepo.fs.server.pojo.ClientListRequest import com.tencent.bkrepo.fs.server.pojo.DailyClientListRequest +import com.tencent.bkrepo.fs.server.request.ClientCreateRequest import com.tencent.bkrepo.fs.server.request.ClientPushMetricsRequest import com.tencent.bkrepo.fs.server.service.ClientService import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder @@ -100,4 +100,4 @@ class ClientHandler( clientService.pushMetrics(pushMetricsRequest) return ReactiveResponseBuilder.success() } -} \ No newline at end of file +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt index c26f7eeb31..8e1dd3682b 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt @@ -28,11 +28,8 @@ package com.tencent.bkrepo.fs.server.handler.service import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.fs.server.request.service.DeleteBlocksRequest import com.tencent.bkrepo.fs.server.request.service.ListBlocksRequest -import com.tencent.bkrepo.fs.server.request.service.RestoreBlocksRequest import com.tencent.bkrepo.fs.server.service.FileNodeService import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder import kotlinx.coroutines.reactor.awaitSingle @@ -42,8 +39,7 @@ import org.springframework.web.reactive.function.server.buildAndAwait class FsNodeHandler( private val rRepositoryClient: RRepositoryClient, - private val fileNodeService: FileNodeService, - private val blockNodeService: BlockNodeService + private val fileNodeService: FileNodeService ) { suspend fun listBlocks(request: ServerRequest): ServerResponse { diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt index fe24eb906a..7ce23a0487 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt @@ -28,8 +28,8 @@ package com.tencent.bkrepo.fs.server.service import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao -import com.tencent.bkrepo.common.metadata.service.blocknode.impl.AbstractBlockNodeService +import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.service.blocknode.impl.RAbstractBlockNodeService import com.tencent.bkrepo.fs.server.api.RRepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import kotlinx.coroutines.reactor.awaitSingle @@ -38,9 +38,9 @@ import kotlinx.coroutines.reactor.awaitSingle * 文件块服务 * */ class BlockNodeServiceImpl( - blockNodeDao: BlockNodeDao, + rBlockNodeDao: RBlockNodeDao, private val rRepositoryClient: RRepositoryClient -) : AbstractBlockNodeService(blockNodeDao) { +) : RAbstractBlockNodeService(rBlockNodeDao) { override suspend fun incFileRef(sha256: String, credentialsKey: String?) { rRepositoryClient.increment(sha256, credentialsKey).awaitSingle() diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt index 121139edd3..71413cd0ec 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt @@ -30,7 +30,7 @@ package com.tencent.bkrepo.fs.server.service import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 @@ -38,19 +38,15 @@ import com.tencent.bkrepo.fs.server.storage.CoStorageManager import com.tencent.bkrepo.repository.pojo.node.NodeDetail class FileNodeService( - private val blockNodeService: BlockNodeService, + private val blockNodeService: RBlockNodeService, private val coStorageManager: CoStorageManager ) { /** * 读取指定范围的文件 * 使用块数据overlay节点数据，当节点数据不存在时，则合并块，同时块之间空的地方补0 - * @param projectId 项目id - * @param repoName 仓库名 - * @param fullPath 节点路径 + * @param nodeDetail 节点详情 * @param storageCredentials 仓库的存储实例 - * @param digest 节点的sha56,当节点不存在时，可以为null - * @param size 节点的大小，当节点不存在时，可以为null * @param range 需要读取的文件范围 * */ suspend fun read( diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt index e0ceb4abdb..aacda5f035 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt @@ -31,7 +31,7 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.model.TBlockNode -import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource @@ -40,7 +40,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext class CoStorageManager( - private val blockNodeService: BlockNodeService, + private val blockNodeService: RBlockNodeService, private val storageService: StorageService ) { diff --git a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt index e27bdcfec3..6e408e410b 100644 --- a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt +++ b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt @@ -4,11 +4,11 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.common.metadata.model.TBlockNode -import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao -import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.com.bkrepo.fs.UT_PROJECT_ID @@ -36,7 +36,7 @@ import reactor.core.publisher.Mono import java.time.LocalDateTime @DataMongoTest -@Import(BlockNodeDao::class) +@Import(RBlockNodeDao::class) @SpringBootConfiguration @EnableAutoConfiguration @TestPropertySource(locations = ["classpath:bootstrap-ut.properties"]) @@ -46,10 +46,10 @@ class BlockNodeServiceTest { lateinit var rRepositoryClient: RRepositoryClient @Autowired - lateinit var blockNodeService: BlockNodeService + lateinit var blockNodeService: RBlockNodeService @Autowired - lateinit var blockNodeDao: BlockNodeDao + lateinit var blockNodeDao: RBlockNodeDao private val storageCredentials = FileSystemCredentials() @BeforeEach diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt index 2ebaf6fc90..7c0a67d41d 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt @@ -34,8 +34,6 @@ import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext import org.springframework.stereotype.Service @Service @@ -45,16 +43,12 @@ class BlockNodeServiceImpl( private val nodeDao: NodeDao ) : AbstractBlockNodeService(blockNodeDao) { - override suspend fun incFileRef(sha256: String, credentialsKey: String?) { - withContext(Dispatchers.IO) { - fileReferenceService.increment(sha256, credentialsKey) - } + override fun incFileRef(sha256: String, credentialsKey: String?) { + fileReferenceService.increment(sha256, credentialsKey) } - override suspend fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail { - return withContext(Dispatchers.IO) { - val node = nodeDao.findNode(projectId, repoName, dstFullPath) ?: throw NodeNotFoundException(dstFullPath) - NodeBaseService.convertToDetail(node)!! - } + override fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail { + val node = nodeDao.findNode(projectId, repoName, dstFullPath) ?: throw NodeNotFoundException(dstFullPath) + return NodeBaseService.convertToDetail(node)!! } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt index 1694091b3d..b9efd89a86 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt @@ -58,7 +58,6 @@ import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeListQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeRestoreUpdate -import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory import org.springframework.dao.DuplicateKeyException import org.springframework.data.mongodb.core.FindAndModifyOptions @@ -179,15 +178,13 @@ open class NodeRestoreSupport( val deletedNode = nodeDao.findAndModify(query, nodeRestoreUpdate(), option, TNode::class.java) if (deletedNode?.sha256 == FAKE_SHA256 || deletedNode?.metadata?.find { it.key == FS_ATTR_KEY } != null) { try { - runBlocking { - blockNodeService.restoreBlocks( - projectId = projectId, - repoName = repoName, - fullPath = fullPath, - nodeCreateDate = deletedNode.createdDate, - nodeDeleteDate = deletedNode.deleted!! - ) - } + blockNodeService.restoreBlocks( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + nodeCreateDate = deletedNode.createdDate, + nodeDeleteDate = deletedNode.deleted!! + ) } catch (e: Exception) { logger.error("restore block resources failed: $projectId/$repoName/$fullPath", e) nodeDao.save(deletedNode) @@ -205,7 +202,7 @@ open class NodeRestoreSupport( with(node) { val query = nodeQuery(projectId, repoName, fullPath) if (node.sha256 == FAKE_SHA256 || node.metadata?.find { it.key == FS_ATTR_KEY } != null) { - runBlocking { blockNodeService.deleteBlocks(projectId, repoName, fullPath) } + blockNodeService.deleteBlocks(projectId, repoName, fullPath) } nodeDao.updateFirst(query, NodeQueryHelper.nodeDeleteUpdate(userId)) } From 822c4d9814a0f045ced3e302abc530a3614f8063 Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Mon, 5 Aug 2024 14:30:21 +0800 Subject: [PATCH 10/11] =?UTF-8?q?feat:=20blockNode=E6=8A=BD=E5=8F=96?= =?UTF-8?q?=E8=87=B3=E5=85=AC=E5=85=B1=E6=A8=A1=E5=9D=97=20#2413?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/api/mongo}/ShardingDocument.kt | 2 +- .../bkrepo/common/api/mongo}/ShardingKey.kt | 2 +- .../artifact-service/build.gradle.kts | 1 + .../manager/NodeResourceFactoryImpl.kt | 6 +- .../manager/resource/FsNodeResource.kt | 16 ++- .../common/metadata/constant/Constant.kt | 3 + .../metadata/MetadataAutoConfiguration.kt | 7 +- .../metadata/condition/ReactiveCondition.kt | 27 +++++ .../metadata/condition/SyncCondition.kt | 27 +++++ .../common/metadata/dao/BlockNodeDao.kt | 27 +++++ .../metadata/dao/MetadataDaoConfiguration.kt | 35 ------ .../common/metadata/model/TBlockNode.kt | 4 +- .../service/blocknode/BlockNodeService.kt | 10 ++ .../service/blocknode/RBlockNodeService.kt | 37 +++++++ .../impl/AbstractBlockNodeService.kt | 64 +++++------ .../impl/DefaultBlockNodeServiceImpl.kt | 26 +++++ .../impl/RAbstractBlockNodeService.kt | 101 ++++++++++------- .../metadata/util/BlockNodeQueryHelper.kt | 103 ++++++++++++++++++ .../common/mongo/reactive/dao/ShardingKey.kt | 48 -------- .../reactive/dao/ShardingMongoReactiveDao.kt | 7 +- .../mongo/dao/sharding/ShardingDocument.kt | 44 -------- .../mongo/dao/sharding/ShardingMongoDao.kt | 2 + .../operate/service/model/TOperateLog.kt | 4 +- .../bkrepo/fs/server/constant/Constants.kt | 5 - .../server/handler/NodeOperationsHandler.kt | 4 +- .../fs/server/service/BlockNodeServiceImpl.kt | 6 +- .../fs/server/service/FileNodeService.kt | 17 +-- .../batch/task/archive/IdleNodeArchiveJob.kt | 2 +- .../job/batch/task/archive/SystemGcJob.kt | 2 +- .../batch/task/clean/DeletedNodeCleanupJob.kt | 2 +- .../job/migrate/executor/BaseTaskExecutor.kt | 2 +- .../job/separation/model/TSeparationNode.kt | 4 +- .../model/TSeparationPackageVersion.kt | 4 +- .../repo/TSeparationMavenMetadataRecord.kt | 4 +- .../service/impl/AbstractHandler.kt | 4 +- .../job/service/impl/ArchiveJobServiceImpl.kt | 2 +- .../migrate/executor/MigrateExecutorTest.kt | 2 +- .../commitedge/CenterClusterReplicator.kt | 2 +- .../replica/type/AbstractReplicaService.kt | 2 +- .../replica/type/edge/EdgeReplicaTaskJob.kt | 2 +- .../bkrepo/repository/model/TFileReference.kt | 4 +- .../tencent/bkrepo/repository/model/TNode.kt | 4 +- .../service/blocknode/BlockNodeServiceImpl.kt | 6 +- .../service/fs/impl/FsServiceImpl.kt | 4 +- .../service/node/impl/NodeBaseService.kt | 4 +- .../service/node/impl/NodeRestoreSupport.kt | 2 +- .../repo/impl/RepositoryServiceImpl.kt | 2 +- .../bkrepo/repository/util/NodeQueryHelper.kt | 2 +- .../repository/service/ServiceBaseTest.kt | 4 +- .../bkrepo/webhook/model/TWebHookLog.kt | 2 +- 50 files changed, 418 insertions(+), 285 deletions(-) rename src/backend/common/{common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao => common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo}/ShardingDocument.kt (96%) rename src/backend/common/{common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding => common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo}/ShardingKey.kt (97%) delete mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt delete mode 100644 src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingKey.kt delete mode 100644 src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingDocument.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingDocument.kt similarity index 96% rename from src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingDocument.kt rename to src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingDocument.kt index ab2231b5a2..2a8bdbb2eb 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingDocument.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingDocument.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.common.mongo.reactive.dao +package com.tencent.bkrepo.common.api.mongo /** * 分表Document diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingKey.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingKey.kt similarity index 97% rename from src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingKey.kt rename to src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingKey.kt index 3085091b08..a7376a2cf8 100644 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingKey.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingKey.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.common.mongo.dao.sharding +package com.tencent.bkrepo.common.api.mongo /** * 分表字段 diff --git a/src/backend/common/common-artifact/artifact-service/build.gradle.kts b/src/backend/common/common-artifact/artifact-service/build.gradle.kts index a043e46bb7..91e92bbf99 100644 --- a/src/backend/common/common-artifact/artifact-service/build.gradle.kts +++ b/src/backend/common/common-artifact/artifact-service/build.gradle.kts @@ -43,6 +43,7 @@ dependencies { api(project(":common:common-operate:operate-service")) api(project(":common:common-stream")) api(project(":common:common-metrics-push")) + api(project(":common:common-metadata:metadata-service")) api("org.springframework.boot:spring-boot-starter-aop") api("io.micrometer:micrometer-registry-prometheus") diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt index a4a87467c4..22fd07c76c 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt @@ -36,11 +36,11 @@ import com.tencent.bkrepo.common.artifact.manager.resource.LocalNodeResource import com.tencent.bkrepo.common.artifact.manager.resource.NodeResource import com.tencent.bkrepo.common.artifact.manager.resource.RemoteNodeResource import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.service.cluster.ClusterInfo import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.replication.api.ClusterNodeClient import com.tencent.bkrepo.replication.exception.ReplicationMessageCode @@ -51,7 +51,7 @@ class NodeResourceFactoryImpl( private val clusterProperties: ClusterProperties, private val storageService: StorageService, private val storageCredentialsClient: StorageCredentialsClient, - private val fsNodeClient: FsNodeClient, + private val blockNodeService: BlockNodeService, private val clusterNodeClient: ClusterNodeClient, private val archiveClient: ArchiveClient, ) : NodeResourceFactory { @@ -71,7 +71,7 @@ class NodeResourceFactoryImpl( ): NodeResource { val digest = nodeInfo.sha256.orEmpty() if (isFsFile(nodeInfo)) { - return FsNodeResource(nodeInfo, fsNodeClient, range, storageService, storageCredentials) + return FsNodeResource(nodeInfo, blockNodeService, range, storageService, storageCredentials) } if (clusterProperties.role == ClusterNodeType.EDGE && clusterProperties.architecture != ClusterArchitecture.COMMIT_EDGE) { diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt index e5d0033cad..d36cf4a595 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt @@ -29,9 +29,10 @@ package com.tencent.bkrepo.common.artifact.manager.resource import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.fs.server.api.FsNodeClient +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo /** @@ -39,20 +40,17 @@ import com.tencent.bkrepo.repository.pojo.node.NodeInfo * */ class FsNodeResource( private val node: NodeInfo, - private val fsNodeClient: FsNodeClient, + private val blockNodeService: BlockNodeService, private val range: Range, private val storageService: StorageService, private val storageCredentials: StorageCredentials? ) : AbstractNodeResource() { override fun getArtifactInputStream(): ArtifactInputStream? { with(node) { - val blocks = fsNodeClient.listBlockResources( - projectId = projectId, - repoName = repoName, - path = fullPath, - startPos = range.start, - endPos = range.end - ).data ?: emptyList() + val blocks = blockNodeService.info( + nodeDetail = NodeDetail(this), + range = range + ) return storageService.load(blocks, range, storageCredentials) } } diff --git a/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt index bc1aa7340c..3f63c95f1d 100644 --- a/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt +++ b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt @@ -29,3 +29,6 @@ package com.tencent.bkrepo.common.metadata.constant const val SHARDING_COUNT = 256 const val ID = "_id" + +const val FAKE_SHA256 = "0000000000000000000000000000000000000000000000000000000000000000" +const val FAKE_MD5 = "00000000000000000000000000000000" diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt index 49ad3b2a54..ccaffc9669 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt @@ -27,14 +27,11 @@ package com.tencent.bkrepo.common.metadata -import com.tencent.bkrepo.common.metadata.dao.MetadataDaoConfiguration import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication +import org.springframework.context.annotation.ComponentScan import org.springframework.context.annotation.Configuration -import org.springframework.context.annotation.Import @Configuration @ConditionalOnWebApplication -@Import( - MetadataDaoConfiguration::class -) +@ComponentScan(basePackages = ["com.tencent.bkrepo.common.metadata"]) class MetadataAutoConfiguration diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt index d2b55647f1..3d8e63eb14 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt @@ -1,3 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + package com.tencent.bkrepo.common.metadata.condition import org.springframework.context.annotation.Condition diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt index 6b17441a28..9514e30e35 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt @@ -1,3 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + package com.tencent.bkrepo.common.metadata.condition import org.springframework.context.annotation.Condition diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt index 779e428c13..3e93f78bab 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt @@ -1,3 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + package com.tencent.bkrepo.common.metadata.dao import com.tencent.bkrepo.common.metadata.condition.SyncCondition diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt deleted file mode 100644 index 5b41522a3f..0000000000 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/MetadataDaoConfiguration.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.common.metadata.dao - -import org.springframework.context.annotation.ComponentScan -import org.springframework.context.annotation.Configuration - -@Configuration -@ComponentScan(basePackages = ["com.tencent.bkrepo.common.metadata.dao"]) -class MetadataDaoConfiguration diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt index a6d47edac1..4ec14a7c41 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt @@ -27,11 +27,11 @@ package com.tencent.bkrepo.common.metadata.model +import com.tencent.bkrepo.common.api.mongo.ShardingDocument import com.tencent.bkrepo.common.metadata.constant.SHARDING_COUNT import com.tencent.bkrepo.common.metadata.model.TBlockNode.Companion.BLOCK_IDX import com.tencent.bkrepo.common.metadata.model.TBlockNode.Companion.BLOCK_IDX_DEF -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingDocument -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingKey import org.springframework.data.mongodb.core.index.CompoundIndex import org.springframework.data.mongodb.core.index.CompoundIndexes import java.time.LocalDateTime diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt index 16e48f2ad8..3557450932 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -30,6 +30,8 @@ package com.tencent.bkrepo.common.metadata.service.blocknode import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.pojo.RegionResource +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import java.time.LocalDateTime interface BlockNodeService { @@ -86,4 +88,12 @@ interface BlockNodeService { nodeCreateDate: LocalDateTime, nodeDeleteDate: LocalDateTime ) + + /** + * 查询节点对应范围的分块资源 + */ + fun info( + nodeDetail: NodeDetail, + range: Range + ): List } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt index f0ccf1015a..15c6f77559 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt @@ -1,8 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + package com.tencent.bkrepo.common.metadata.service.blocknode import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.pojo.RegionResource +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import java.time.LocalDateTime interface RBlockNodeService { @@ -58,4 +87,12 @@ interface RBlockNodeService { nodeCreateDate: LocalDateTime, nodeDeleteDate: LocalDateTime ) + + /** + * 查询节点对应范围的分块资源 + */ + suspend fun info( + nodeDetail: NodeDetail, + range: Range + ): List } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt index ec143adb29..edb6cfeb7a 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt @@ -27,24 +27,22 @@ package com.tencent.bkrepo.common.metadata.service.blocknode.impl -import com.tencent.bkrepo.common.api.util.EscapeUtils import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.constant.ID import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.util.BlockNodeQueryHelper import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.slf4j.LoggerFactory -import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.and -import org.springframework.data.mongodb.core.query.gt import org.springframework.data.mongodb.core.query.isEqualTo -import org.springframework.data.mongodb.core.query.lt -import org.springframework.data.mongodb.core.query.where import java.time.LocalDateTime abstract class AbstractBlockNodeService( @@ -67,16 +65,7 @@ abstract class AbstractBlockNodeService( fullPath: String, createdDate: String ): List { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - .and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) - .norOperator( - TBlockNode::startPos.gt(range.end), - TBlockNode::endPos.lt(range.start) - ) - val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) + val query = BlockNodeQueryHelper.listQuery(projectId, repoName, fullPath, createdDate, range) return blockNodeDao.find(query) } @@ -85,11 +74,8 @@ abstract class AbstractBlockNodeService( repoName: String, fullPath: String ) { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val update = Update().set(TBlockNode::deleted.name, LocalDateTime.now()) + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + val update = BlockNodeQueryHelper.deleteUpdate() blockNodeDao.updateMulti(Query(criteria), update) logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") } @@ -97,10 +83,7 @@ abstract class AbstractBlockNodeService( override fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) if (nodeDetail.folder) { - val criteria = where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, true) val blocks = blockNodeDao.find(Query(criteria)) blocks.forEach { val update = Update().set(TBlockNode::nodeFullPath.name, it.nodeFullPath.replace(fullPath, dstFullPath)) @@ -108,11 +91,8 @@ abstract class AbstractBlockNodeService( blockNodeDao.updateMulti(query, update) } } else { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val update = Update().set(TBlockNode::nodeFullPath.name, dstFullPath) + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + val update = BlockNodeQueryHelper.moveUpdate(dstFullPath) blockNodeDao.updateMulti(Query(criteria), update) } logger.info("Move node[$projectId/$repoName$fullPath] to node[$projectId/$repoName$dstFullPath] success.") @@ -127,19 +107,33 @@ abstract class AbstractBlockNodeService( nodeCreateDate: LocalDateTime, nodeDeleteDate: LocalDateTime ) { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::createdDate).gt(nodeCreateDate).lt(nodeDeleteDate) - val update = Update().set(TBlockNode::deleted.name, null) + val criteria = + BlockNodeQueryHelper.deletedCriteria(projectId, repoName, fullPath, nodeCreateDate, nodeDeleteDate) + val update = BlockNodeQueryHelper.restoreUpdate() val result = blockNodeDao.updateMulti(Query(criteria), update) logger.info("Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + "between $nodeCreateDate and $nodeDeleteDate success.") } + override fun info(nodeDetail: NodeDetail, range: Range): List { + with(nodeDetail) { + val blocks = listBlocks(range, projectId, repoName, fullPath, createdDate) + val blockResources = mutableListOf() + if (sha256 != null && sha256 != FAKE_SHA256) { + val nodeData = RegionResource(sha256!!, 0, size, 0, size) + blockResources.add(nodeData) + } + blocks.forEach { + val res = RegionResource(it.sha256, it.startPos, it.size, 0, it.size) + blockResources.add(res) + } + return blockResources + } + } + abstract fun incFileRef(sha256: String, credentialsKey: String?) - abstract fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail + abstract fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail companion object { private val logger = LoggerFactory.getLogger(AbstractBlockNodeService::class.java) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt new file mode 100644 index 0000000000..688bd865b4 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt @@ -0,0 +1,26 @@ +package com.tencent.bkrepo.common.metadata.service.blocknode.impl + +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException +import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.repository.api.FileReferenceClient +import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Service + +@Service +@Conditional(SyncCondition::class) +class DefaultBlockNodeServiceImpl( + blockNodeDao: BlockNodeDao, + private val fileReferenceClient: FileReferenceClient, + private val nodeClient: NodeClient +) : AbstractBlockNodeService(blockNodeDao) { + override fun incFileRef(sha256: String, credentialsKey: String?) { + fileReferenceClient.increment(sha256, credentialsKey) + } + + override fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { + return nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: throw NodeNotFoundException(fullPath) + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt index 6fb28d1660..4e48e22f5b 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt @@ -1,23 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + package com.tencent.bkrepo.common.metadata.service.blocknode.impl -import com.tencent.bkrepo.common.api.util.EscapeUtils import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.constant.ID import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService +import com.tencent.bkrepo.common.metadata.util.BlockNodeQueryHelper import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.slf4j.LoggerFactory -import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query -import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.and -import org.springframework.data.mongodb.core.query.gt import org.springframework.data.mongodb.core.query.isEqualTo -import org.springframework.data.mongodb.core.query.lt -import org.springframework.data.mongodb.core.query.where import java.time.LocalDateTime abstract class RAbstractBlockNodeService( @@ -40,16 +64,7 @@ abstract class RAbstractBlockNodeService( fullPath: String, createdDate: String ): List { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - .and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) - .norOperator( - TBlockNode::startPos.gt(range.end), - TBlockNode::endPos.lt(range.start) - ) - val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) + val query = BlockNodeQueryHelper.listQuery(projectId, repoName, fullPath, createdDate, range) return rBlockNodeDao.find(query) } @@ -58,11 +73,8 @@ abstract class RAbstractBlockNodeService( repoName: String, fullPath: String ) { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val update = Update().set(TBlockNode::deleted.name, LocalDateTime.now()) + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + val update = BlockNodeQueryHelper.deleteUpdate() rBlockNodeDao.updateMulti(Query(criteria), update) logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") } @@ -70,29 +82,22 @@ abstract class RAbstractBlockNodeService( override suspend fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) if (nodeDetail.folder) { - val criteria = where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, true) val blocks = rBlockNodeDao.find(Query(criteria)) blocks.forEach { - val update = Update().set(TBlockNode::nodeFullPath.name, it.nodeFullPath.replace(fullPath, dstFullPath)) + val update = BlockNodeQueryHelper.moveUpdate(it.nodeFullPath.replace(fullPath, dstFullPath)) val query = Query(Criteria.where(ID).isEqualTo(it.id).and(TBlockNode::repoName).isEqualTo(repoName)) rBlockNodeDao.updateMulti(query, update) } } else { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val update = Update().set(TBlockNode::nodeFullPath.name, dstFullPath) + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + val update = BlockNodeQueryHelper.moveUpdate(dstFullPath) rBlockNodeDao.updateMulti(Query(criteria), update) } logger.info("Move node[$projectId/$repoName$fullPath] to node[$projectId/$repoName$dstFullPath] success.") } - override suspend fun restoreBlocks( projectId: String, repoName: String, @@ -100,19 +105,35 @@ abstract class RAbstractBlockNodeService( nodeCreateDate: LocalDateTime, nodeDeleteDate: LocalDateTime ) { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::createdDate).gt(nodeCreateDate).lt(nodeDeleteDate) - val update = Update().set(TBlockNode::deleted.name, null) + val criteria = + BlockNodeQueryHelper.deletedCriteria(projectId, repoName, fullPath, nodeCreateDate, nodeDeleteDate) + val update = BlockNodeQueryHelper.restoreUpdate() val result = rBlockNodeDao.updateMulti(Query(criteria), update) - logger.info("Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + - "between $nodeCreateDate and $nodeDeleteDate success.") + logger.info( + "Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + + "between $nodeCreateDate and $nodeDeleteDate success." + ) + } + + override suspend fun info(nodeDetail: NodeDetail, range: Range): List { + with(nodeDetail) { + val blocks = listBlocks(range, projectId, repoName, fullPath, createdDate) + val blockResources = mutableListOf() + if (sha256 != null && sha256 != FAKE_SHA256) { + val nodeData = RegionResource(sha256!!, 0, size, 0, size) + blockResources.add(nodeData) + } + blocks.forEach { + val res = RegionResource(it.sha256, it.startPos, it.size, 0, it.size) + blockResources.add(res) + } + return blockResources + } } abstract suspend fun incFileRef(sha256: String, credentialsKey: String?) - abstract suspend fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail + abstract suspend fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail companion object { private val logger = LoggerFactory.getLogger(RAbstractBlockNodeService::class.java) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt new file mode 100644 index 0000000000..396e6b14e5 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.util + +import com.tencent.bkrepo.common.api.util.EscapeUtils +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.gt +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.lt +import org.springframework.data.mongodb.core.query.where +import java.time.LocalDateTime + +object BlockNodeQueryHelper { + + fun listQuery( + projectId: String, + repoName: String, + fullPath: String, + createdDate: String, + range: Range + ): Query { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId).isEqualTo(projectId) + .and(TBlockNode::repoName).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + .and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) + .norOperator( + TBlockNode::startPos.gt(range.end), + TBlockNode::endPos.lt(range.start) + ) + val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) + return query + } + + fun fullPathCriteria(projectId: String, repoName: String, fullPath: String, deep: Boolean): Criteria { + val criteria = if (deep) { + where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") + } else { + where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + } + return criteria + .and(TBlockNode::projectId).isEqualTo(projectId) + .and(TBlockNode::repoName).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + } + + fun deletedCriteria( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ): Criteria { + return where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::createdDate).gt(nodeCreateDate).lt(nodeDeleteDate) + } + + fun deleteUpdate(): Update { + return Update().set(TBlockNode::deleted.name, LocalDateTime.now()) + } + + fun moveUpdate(dstFullPath: String): Update { + return Update().set(TBlockNode::nodeFullPath.name, dstFullPath) + } + + fun restoreUpdate(): Update { + return Update().set(TBlockNode::deleted.name, null) + } + +} diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingKey.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingKey.kt deleted file mode 100644 index 59dc10b1d7..0000000000 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingKey.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.tencent.bkrepo.common.mongo.reactive.dao - -/** - * 分表字段 - */ -@Retention(AnnotationRetention.RUNTIME) -@Target(AnnotationTarget.FIELD) -annotation class ShardingKey( - /** - * 分表字段 - */ - val column: String = "", - /** - * 分表数，power of 2 - */ - val count: Int = 1 -) diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt index 2dc45cacf5..e9351b3a0b 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt +++ b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt @@ -28,10 +28,10 @@ package com.tencent.bkrepo.common.mongo.reactive.dao import com.mongodb.BasicDBList +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.common.mongo.util.MongoIndexResolver import com.tencent.bkrepo.common.mongo.util.ShardingUtils -import java.lang.reflect.Field -import javax.annotation.PostConstruct import org.apache.commons.lang3.reflect.FieldUtils import org.bson.Document import org.slf4j.LoggerFactory @@ -40,8 +40,9 @@ import org.springframework.beans.factory.annotation.Value import org.springframework.core.annotation.AnnotationUtils import org.springframework.data.mongodb.core.ReactiveMongoOperations import org.springframework.data.mongodb.core.index.IndexDefinition -import org.springframework.data.mongodb.core.indexOps import org.springframework.data.mongodb.core.query.Query +import java.lang.reflect.Field +import javax.annotation.PostConstruct abstract class ShardingMongoReactiveDao : AbstractMongoReactiveDao() { diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt deleted file mode 100644 index 78dfa669de..0000000000 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.tencent.bkrepo.common.mongo.dao.sharding - -/** - * 分表Document - */ -@Retention(AnnotationRetention.RUNTIME) -@Target(AnnotationTarget.CLASS) -annotation class ShardingDocument( - /** - * collection name - */ - val collection: String = "" -) diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt index a771a1075c..ab19360ee1 100644 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt +++ b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt @@ -32,6 +32,8 @@ package com.tencent.bkrepo.common.mongo.dao.sharding import com.mongodb.BasicDBList +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.common.mongo.dao.AbstractMongoDao import com.tencent.bkrepo.common.mongo.dao.util.MongoIndexResolver import com.tencent.bkrepo.common.mongo.dao.util.sharding.ShardingUtils diff --git a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TOperateLog.kt b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TOperateLog.kt index b80bff887c..e3a7a99c48 100644 --- a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TOperateLog.kt +++ b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TOperateLog.kt @@ -31,8 +31,8 @@ package com.tencent.bkrepo.common.operate.service.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.common.operate.service.model.TOperateLog.Companion.RESOURCE_KEY_IDX import com.tencent.bkrepo.common.operate.service.model.TOperateLog.Companion.RESOURCE_KEY_IDX_DEF import org.springframework.data.mongodb.core.index.CompoundIndex diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt index 50a8db58a7..d2f9c3b457 100644 --- a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt +++ b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt @@ -33,8 +33,3 @@ const val JWT_CLAIMS_REPOSITORY = "repository" const val JWT_CLAIMS_PERMIT = "permit" const val FS_ATTR_KEY = "fs:attr" - -const val FAKE_SHA256 = "0000000000000000000000000000000000000000000000000000000000000000" -const val FAKE_MD5 = "00000000000000000000000000000000" - -const val ID = "_id" diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt index 7d2f8519ae..bc20036819 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt @@ -30,11 +30,11 @@ package com.tencent.bkrepo.fs.server.handler import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.storage.core.overlay.OverlayRangeUtils import com.tencent.bkrepo.fs.server.api.RGenericClient import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.fs.server.constant.FAKE_MD5 -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder import com.tencent.bkrepo.fs.server.model.NodeAttribute diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt index 7ce23a0487..2523c43a5e 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt @@ -46,8 +46,8 @@ class BlockNodeServiceImpl( rRepositoryClient.increment(sha256, credentialsKey).awaitSingle() } - override suspend fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail { - return rRepositoryClient.getNodeDetail(projectId, repoName, dstFullPath).awaitSingle().data - ?: throw NodeNotFoundException(dstFullPath) + override suspend fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { + return rRepositoryClient.getNodeDetail(projectId, repoName, fullPath).awaitSingle().data + ?: throw NodeNotFoundException(fullPath) } } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt index 71413cd0ec..c62022e7a2 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt @@ -33,7 +33,6 @@ import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.fs.server.storage.CoStorageManager import com.tencent.bkrepo.repository.pojo.node.NodeDetail @@ -54,7 +53,7 @@ class FileNodeService( storageCredentials: StorageCredentials?, range: Range ): ArtifactInputStream? { - val blocks = info(nodeDetail, range) + val blocks = blockNodeService.info(nodeDetail, range) return coStorageManager.loadArtifactInputStream(blocks, range, storageCredentials) } @@ -62,19 +61,7 @@ class FileNodeService( nodeDetail: NodeDetail, range: Range ): List { - with(nodeDetail) { - val blocks = blockNodeService.listBlocks(range, projectId, repoName, fullPath, createdDate) - val blockResources = mutableListOf() - if (sha256 != null && sha256 != FAKE_SHA256) { - val nodeData = RegionResource(sha256!!, 0, size, 0, size) - blockResources.add(nodeData) - } - blocks.forEach { - val res = RegionResource(it.sha256, it.startPos, it.size, 0, it.size) - blockResources.add(res) - } - return blockResources - } + return blockNodeService.info(nodeDetail, range) } suspend fun deleteNodeBlocks(projectId: String, repoName: String, nodeFullPath: String) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt index 652c05dc55..e9769d0f3d 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt @@ -30,8 +30,8 @@ package com.tencent.bkrepo.job.batch.task.archive import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.constant.ArchiveStorageClass import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob import com.tencent.bkrepo.job.batch.context.NodeContext diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/SystemGcJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/SystemGcJob.kt index 1f225c805a..5684aa9130 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/SystemGcJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/SystemGcJob.kt @@ -33,12 +33,12 @@ import com.tencent.bkrepo.archive.request.CompressFileRequest import com.tencent.bkrepo.common.api.collection.groupBySimilar import com.tencent.bkrepo.common.api.constant.retry import com.tencent.bkrepo.common.api.util.HumanReadable +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.common.service.exception.RemoteErrorCodeException import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.DefaultContextJob import com.tencent.bkrepo.job.batch.base.JobContext diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt index 1464c76a63..f2a245312b 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt @@ -34,9 +34,9 @@ import com.google.common.util.concurrent.UncheckedExecutionException import com.mongodb.client.result.DeleteResult import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.DELETED_DATE import com.tencent.bkrepo.job.NAME import com.tencent.bkrepo.job.PROJECT diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt index cd7c859a56..9fa97b33f2 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt @@ -28,10 +28,10 @@ package com.tencent.bkrepo.job.migrate.executor import com.tencent.bkrepo.common.api.util.HumanReadable +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.actuator.ActuatorConfiguration.Companion.SERVICE_INSTANCE_ID import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.monitor.measureThroughput -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.migrate.dao.MigrateFailedNodeDao import com.tencent.bkrepo.job.migrate.dao.MigrateRepoStorageTaskDao diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationNode.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationNode.kt index ac63fec108..967c04bdff 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationNode.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationNode.kt @@ -54,8 +54,8 @@ package com.tencent.bkrepo.job.separation.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.job.separation.model.TSeparationNode.Companion.SEPARATION_FOLDER_IDX import com.tencent.bkrepo.job.separation.model.TSeparationNode.Companion.SEPARATION_FOLDER_IDX_DEF import com.tencent.bkrepo.job.separation.model.TSeparationNode.Companion.SEPARATION_FULL_PATH_IDX diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationPackageVersion.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationPackageVersion.kt index 6f9cb4b13f..0c33d8c9c5 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationPackageVersion.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationPackageVersion.kt @@ -54,8 +54,8 @@ package com.tencent.bkrepo.job.separation.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.job.separation.model.TSeparationPackageVersion.Companion.SEPARATION_VERSION_NAME_IDX import com.tencent.bkrepo.job.separation.model.TSeparationPackageVersion.Companion.SEPARATION_VERSION_NAME_IDX_DEF import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/repo/TSeparationMavenMetadataRecord.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/repo/TSeparationMavenMetadataRecord.kt index 99fe6f6b23..68d21ab8a7 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/repo/TSeparationMavenMetadataRecord.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/repo/TSeparationMavenMetadataRecord.kt @@ -54,8 +54,8 @@ package com.tencent.bkrepo.job.separation.model.repo -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import org.springframework.data.mongodb.core.index.CompoundIndex import org.springframework.data.mongodb.core.index.CompoundIndexes import java.time.LocalDateTime diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt index 588ea5dfd4..682b756cbb 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt @@ -29,8 +29,8 @@ package com.tencent.bkrepo.job.separation.service.impl import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.constant.ID -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.separation.dao.SeparationFailedRecordDao import com.tencent.bkrepo.job.separation.dao.SeparationTaskDao import com.tencent.bkrepo.job.separation.pojo.NodeFilterInfo @@ -218,4 +218,4 @@ open class AbstractHandler( companion object { private val logger = LoggerFactory.getLogger(AbstractHandler::class.java) } -} \ No newline at end of file +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt index ec6378970d..7d54f2d38c 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt @@ -4,9 +4,9 @@ import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.constant.ArchiveStorageClass import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.archive.request.UncompressFileRequest +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.common.query.util.MongoEscapeUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.BATCH_SIZE import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.context.NodeContext diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt index 65a6ec3c7d..b6bc6460f4 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt @@ -1,6 +1,6 @@ package com.tencent.bkrepo.job.migrate.executor -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.job.UT_PROJECT_ID import com.tencent.bkrepo.job.UT_REPO_NAME import com.tencent.bkrepo.job.UT_STORAGE_CREDENTIALS_KEY diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt index ad22b0261a..5b045fc5a8 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt @@ -27,9 +27,9 @@ package com.tencent.bkrepo.replication.replica.replicator.commitedge +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.replica.context.ReplicaContext diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt index da14276aa5..a0de01d3ea 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt @@ -32,7 +32,7 @@ import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.metrics.ReplicationRecord import com.tencent.bkrepo.replication.pojo.record.ExecutionResult diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt index f2158cdd37..abcbbb1cd9 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt @@ -33,12 +33,12 @@ import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.PackageNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition import com.tencent.bkrepo.common.service.feign.FeignClientFactory import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace import com.tencent.bkrepo.common.service.util.UrlUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.replication.api.cluster.ClusterReplicaTaskClient import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt index b249753af7..cc9750d100 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt @@ -31,8 +31,8 @@ package com.tencent.bkrepo.repository.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.repository.constant.SHARDING_COUNT import org.springframework.data.mongodb.core.index.CompoundIndex import org.springframework.data.mongodb.core.index.CompoundIndexes diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt index 7023dc5b9d..cf6f57bee1 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt @@ -31,8 +31,8 @@ package com.tencent.bkrepo.repository.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.repository.constant.SHARDING_COUNT import com.tencent.bkrepo.repository.model.TNode.Companion.ARCHIVED_IDX import com.tencent.bkrepo.repository.model.TNode.Companion.ARCHIVED_IDX_DEF diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt index 7c0a67d41d..bd8d5b9f31 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt @@ -34,9 +34,11 @@ import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService +import org.springframework.context.annotation.Primary import org.springframework.stereotype.Service @Service +@Primary class BlockNodeServiceImpl( blockNodeDao: BlockNodeDao, private val fileReferenceService: FileReferenceService, @@ -47,8 +49,8 @@ class BlockNodeServiceImpl( fileReferenceService.increment(sha256, credentialsKey) } - override fun getNodeDetail(projectId: String, repoName: String, dstFullPath: String): NodeDetail { - val node = nodeDao.findNode(projectId, repoName, dstFullPath) ?: throw NodeNotFoundException(dstFullPath) + override fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { + val node = nodeDao.findNode(projectId, repoName, fullPath) ?: throw NodeNotFoundException(fullPath) return NodeBaseService.convertToDetail(node)!! } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt index d8575b05aa..edaff62bc8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt @@ -31,10 +31,10 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition -import com.tencent.bkrepo.fs.server.constant.FAKE_MD5 -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.model.TNode import com.tencent.bkrepo.repository.pojo.node.NodeDetail diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt index 5674e117fa..6758011107 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt @@ -42,6 +42,8 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.query.model.Sort @@ -52,8 +54,6 @@ import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publi import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.constant.BinderType import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.constant.FAKE_MD5 -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.dao.NodeDao diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt index b9efd89a86..111193026a 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt @@ -35,9 +35,9 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils.isRoot +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.model.TNode diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt index 386499d830..69d6aaa315 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt @@ -49,6 +49,7 @@ import com.tencent.bkrepo.common.artifact.pojo.configuration.composite.ProxyConf import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.virtual.VirtualConfiguration +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.dao.AbstractMongoDao.Companion.ID import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.security.util.RsaUtils @@ -57,7 +58,6 @@ import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.RepositoryDao import com.tencent.bkrepo.repository.dao.repository.ProjectMetricsRepository diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt index d0cb05bb8f..f634a784ef 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt @@ -32,9 +32,9 @@ import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.path.PathUtils.escapeRegex import com.tencent.bkrepo.common.artifact.path.PathUtils.toFullPath import com.tencent.bkrepo.common.artifact.path.PathUtils.toPath +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.util.MongoEscapeUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.model.TNode import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeListOption diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt index d7022a6902..311350a10b 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt @@ -43,6 +43,7 @@ import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager @@ -97,7 +98,8 @@ import org.springframework.test.context.TestPropertySource HttpAuthProperties::class, SpringContextUtils::class, NodeDao::class, - RouterControllerProperties::class + RouterControllerProperties::class, + BlockNodeDao::class ) @ComponentScan("com.tencent.bkrepo.repository.service") @TestPropertySource(locations = ["classpath:bootstrap-ut.properties", "classpath:center-ut.properties"]) diff --git a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/model/TWebHookLog.kt b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/model/TWebHookLog.kt index 07905bad86..010bc3ac53 100644 --- a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/model/TWebHookLog.kt +++ b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/model/TWebHookLog.kt @@ -28,7 +28,7 @@ package com.tencent.bkrepo.webhook.model import com.tencent.bkrepo.common.artifact.event.base.EventType -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.webhook.constant.WebHookRequestStatus import org.springframework.data.mongodb.core.mapping.Document import java.time.LocalDateTime From 8901e522f40928b11e7b84c9a69e8b8818cf127e Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Tue, 6 Aug 2024 09:53:29 +0800 Subject: [PATCH 11/11] =?UTF-8?q?feat:=20blockNode=E6=8A=BD=E5=8F=96?= =?UTF-8?q?=E8=87=B3=E5=85=AC=E5=85=B1=E6=A8=A1=E5=9D=97=20#2413?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/config/FsNodeDefaultImpl.kt | 51 ------------------- .../bkrepo/fs/server/api/FsNodeClient.kt | 51 ------------------- .../repository/service/ServiceBaseTest.kt | 4 -- 3 files changed, 106 deletions(-) delete mode 100644 src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt delete mode 100644 src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt diff --git a/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt b/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt deleted file mode 100644 index b405a2c13e..0000000000 --- a/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.config - -import com.tencent.bkrepo.common.api.exception.MethodNotAllowedException -import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.storage.pojo.RegionResource -import com.tencent.bkrepo.fs.server.api.FsNodeClient -import org.springframework.stereotype.Service - -/** - * fs-server提供的是reactive的实现，与boot-assembly冲突， - * 所以这里提供一个默认实现 - * */ -@Service -class FsNodeDefaultImpl : FsNodeClient { - override fun listBlockResources( - projectId: String, - repoName: String, - path: String, - startPos: Long, - endPos: Long - ): Response> { - throw MethodNotAllowedException() - } -} diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt deleted file mode 100644 index 76fd96a2cf..0000000000 --- a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.fs.server.api - -import com.tencent.bkrepo.common.api.constant.FS_SERVER_SERVICE_NAME -import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.storage.pojo.RegionResource -import org.springframework.cloud.openfeign.FeignClient -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam - -@FeignClient(FS_SERVER_SERVICE_NAME, contextId = "BlockClient", primary = false) -@RequestMapping("/service/block") -interface FsNodeClient { - - @GetMapping("/list/{projectId}/{repoName}") - fun listBlockResources( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam path: String, - @RequestParam startPos: Long, - @RequestParam endPos: Long - ): Response> -} diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt index 311350a10b..928b6455c7 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt @@ -53,7 +53,6 @@ import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.UT_PROJECT_ID import com.tencent.bkrepo.repository.UT_REPO_DESC import com.tencent.bkrepo.repository.UT_REPO_DISPLAY @@ -135,9 +134,6 @@ open class ServiceBaseTest { @MockBean lateinit var routerControllerClient: RouterControllerClient - @MockBean - lateinit var fsNodeClient: FsNodeClient - @Autowired lateinit var springContextUtils: SpringContextUtils