Issue 2413 new

.github/workflows/assembly-check.yml

@@ -0,0 +1,46 @@
+name: Build assembly application and test
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ master ]
+    paths:
+    - 'src/backend/**'
+  pull_request:
+    branches: [ master, release-* ]
+    paths:
+    - 'src/backend/**'
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build-and-test:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Cache Gradle
+      uses: actions/cache@v1
+      with:
+        path: ~/.gradle/caches
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
+        restore-keys: |
+          ${{ runner.os }}-gradle-
+    - name: Start MongoDB
+      run: |
+        docker run -d --name mongodb -p 27017:27017 mongo
+    - name: Gradle Build and test
+      working-directory: src/backend/
+      run: |
+        ./gradlew clean
+        ./gradlew boot-assembly:build -x test
+        java -jar -Dlogging.path=boot-assembly/build/libs/bkrepo/logs -Dstorage.filesystem.path=boot-assembly/build/libs/bkrepo/bkrepo boot-assembly/build/libs/boot-assembly-1.0.0-RELEASE.jar &
+        sleep 60
+        curl http://localhost:8080/helm/blueking/helm-local/index.yaml

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/configuration/ScannerConfiguration.kt

@@ -30,7 +30,7 @@ package com.tencent.bkrepo.analyst.configuration
 import com.tencent.bkrepo.analysis.executor.api.ExecutorClient
 import com.tencent.bkrepo.analyst.dispatcher.SubtaskDispatcherFactory
 import com.tencent.bkrepo.analyst.dispatcher.SubtaskPoller
-import com.tencent.bkrepo.analyst.event.ScanEventConsumer
+import com.tencent.bkrepo.analyst.event.AnalystScanEventConsumer
 import com.tencent.bkrepo.analyst.service.ExecutionClusterService
 import com.tencent.bkrepo.analyst.service.ScannerService
 import com.tencent.bkrepo.analyst.service.impl.OperateLogServiceImpl
@@ -91,10 +91,10 @@ class ScannerConfiguration {
 
     @Bean("scanEventConsumer")
     fun scanEventConsumer(
-        scanEventConsumer: ScanEventConsumer
+        analystScanEventConsumer: AnalystScanEventConsumer
     ): Consumer<ArtifactEvent> {
         return Consumer {
-            scanEventConsumer.accept(it)
+            analystScanEventConsumer.accept(it)
         }
     }
 }

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/ScanEventConsumer.kt → src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/event/AnalystScanEventConsumer.kt

@@ -63,7 +63,7 @@ import org.springframework.stereotype.Component
  * 对应binding name为scanEventConsumer-in-0
  */
 @Component
-class ScanEventConsumer(
+class AnalystScanEventConsumer(
     private val spdxLicenseService: SpdxLicenseService,
     private val scanService: ScanService,
     private val scannerService: ScannerService,
@@ -287,6 +287,6 @@ class ScanEventConsumer(
     }
 
     companion object {
-        private val logger = LoggerFactory.getLogger(ScanEventConsumer::class.java)
+        private val logger = LoggerFactory.getLogger(AnalystScanEventConsumer::class.java)
     }
 }

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/archive/ArchiveManager.kt

@@ -39,7 +39,6 @@ import java.util.function.Function
 class ArchiveManager(
     private val archiveProperties: ArchiveProperties,
     private val fileProvider: PriorityFileProvider,
-    private val storageService: StorageService,
 ) : Function<TArchiveFile, Mono<TaskResult>> {
     @Autowired
     @Lazy
@@ -49,6 +48,10 @@ class ArchiveManager(
     @Lazy
     private lateinit var archiveFileRepository: ArchiveFileRepository
 
+    @Autowired
+    @Lazy
+    private lateinit var storageService: StorageService
+
     private val tika = Tika()
     private val compressPool = ArchiveUtils.newFixedAndCachedThreadPool(
         archiveProperties.compress.compressThreads,

src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt

@@ -42,7 +42,6 @@ import java.util.function.Function
 @Component
 class BDZipManager(
     private val archiveProperties: ArchiveProperties,
-    private val storageService: StorageService,
     private val fileProvider: PriorityFileProvider,
 ) : Function<TCompressFile, Mono<TaskResult>> {
     @Autowired
@@ -53,6 +52,11 @@ class BDZipManager(
     @Lazy
     private lateinit var fileReferenceClient: FileReferenceClient
 
+    @Autowired
+    @Lazy
+    private lateinit var storageService: StorageService
+
+
     @Autowired
     @Lazy
     private lateinit var compressFileRepository: CompressFileRepository

src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/model/TRepoAuthConfig.kt

@@ -40,7 +40,7 @@ import java.time.LocalDateTime
 @Document("repo_auth_mode")
 @CompoundIndexes(
     CompoundIndex(name = "repo_idx", def = "{'projectId': 1, 'repoName': 1}", background = true, unique = true),
-    CompoundIndex(name = "access_ctrl_idx", def = "{'accessControlMode': 1}", background = true)
+    CompoundIndex(name = "access_ctrl_mode_idx", def = "{'accessControlMode': 1}", background = true)
 )
 data class TRepoAuthConfig(
     var id: String? = null,

src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/bkdevops/DevopsPermissionServiceImpl.kt

@@ -305,4 +305,4 @@ class DevopsPermissionServiceImpl constructor(
     companion object {
         private val logger = LoggerFactory.getLogger(DevopsPermissionServiceImpl::class.java)
     }
-}
+}

src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt → src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingDocument.kt

@@ -29,7 +29,7 @@
  * SOFTWARE.
  */
 
-package com.tencent.bkrepo.common.mongo.dao.sharding
+package com.tencent.bkrepo.common.api.mongo
 
 /**
  * 分表Document

src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingKey.kt → src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingKey.kt

@@ -29,7 +29,7 @@
  * SOFTWARE.
  */
 
-package com.tencent.bkrepo.common.mongo.dao.sharding
+package com.tencent.bkrepo.common.api.mongo
 
 /**
  * 分表字段

src/backend/common/common-artifact/artifact-service/build.gradle.kts

@@ -43,6 +43,7 @@ dependencies {
     api(project(":common:common-operate:operate-service"))
     api(project(":common:common-stream"))
     api(project(":common:common-metrics-push"))
+    api(project(":common:common-metadata:metadata-service"))
 
     api("org.springframework.boot:spring-boot-starter-aop")
     api("io.micrometer:micrometer-registry-prometheus")

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/listener/ArtifactTransferListener.kt

@@ -37,6 +37,7 @@ import com.tencent.bkrepo.common.artifact.constant.DEFAULT_STORAGE_KEY
 import com.tencent.bkrepo.common.artifact.event.ArtifactReceivedEvent
 import com.tencent.bkrepo.common.artifact.event.ArtifactResponseEvent
 import com.tencent.bkrepo.common.artifact.event.ChunkArtifactTransferEvent
+import com.tencent.bkrepo.common.artifact.hash.md5
 import com.tencent.bkrepo.common.artifact.metrics.ArtifactCacheMetrics
 import com.tencent.bkrepo.common.artifact.metrics.ArtifactMetrics
 import com.tencent.bkrepo.common.artifact.metrics.ArtifactMetricsProperties
@@ -106,7 +107,8 @@ class ArtifactTransferListener(
                     host = artifactMetricsProperties.host,
                     builderAgentList = artifactMetricsProperties.builderAgentList,
                     clientAgentList = artifactMetricsProperties.clientAgentList
-                ).name
+                ).name,
+                userId = SecurityUtils.getUserId().md5()
             )
             if (SecurityUtils.getUserId() != SYSTEM_USER) {
                 projectUsageStatisticsService.inc(projectId = projectId, receivedBytes = throughput.bytes)
@@ -151,7 +153,8 @@ class ArtifactTransferListener(
                     host = artifactMetricsProperties.host,
                     builderAgentList = artifactMetricsProperties.builderAgentList,
                     clientAgentList = artifactMetricsProperties.clientAgentList
-                ).name
+                ).name,
+                userId = SecurityUtils.getUserId().md5()
             )
             if (SecurityUtils.getUserId() != SYSTEM_USER) {
                 projectUsageStatisticsService.inc(projectId = projectId, responseBytes = throughput.bytes)
@@ -207,7 +210,8 @@ class ArtifactTransferListener(
                     host = artifactMetricsProperties.host,
                     builderAgentList = artifactMetricsProperties.builderAgentList,
                     clientAgentList = artifactMetricsProperties.clientAgentList
-                ).name
+                ).name,
+                userId = SecurityUtils.getUserId().md5()
             )
             if (artifactMetricsProperties.collectByLog) {
                 logger.info(

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt

@@ -36,11 +36,11 @@ import com.tencent.bkrepo.common.artifact.manager.resource.LocalNodeResource
 import com.tencent.bkrepo.common.artifact.manager.resource.NodeResource
 import com.tencent.bkrepo.common.artifact.manager.resource.RemoteNodeResource
 import com.tencent.bkrepo.common.artifact.stream.Range
+import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService
 import com.tencent.bkrepo.common.service.cluster.ClusterInfo
 import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties
 import com.tencent.bkrepo.common.storage.core.StorageService
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
-import com.tencent.bkrepo.fs.server.api.FsNodeClient
 import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY
 import com.tencent.bkrepo.replication.api.ClusterNodeClient
 import com.tencent.bkrepo.replication.exception.ReplicationMessageCode
@@ -51,7 +51,7 @@ class NodeResourceFactoryImpl(
     private val clusterProperties: ClusterProperties,
     private val storageService: StorageService,
     private val storageCredentialsClient: StorageCredentialsClient,
-    private val fsNodeClient: FsNodeClient,
+    private val blockNodeService: BlockNodeService,
     private val clusterNodeClient: ClusterNodeClient,
     private val archiveClient: ArchiveClient,
 ) : NodeResourceFactory {
@@ -71,7 +71,7 @@ class NodeResourceFactoryImpl(
     ): NodeResource {
         val digest = nodeInfo.sha256.orEmpty()
         if (isFsFile(nodeInfo)) {
-            return FsNodeResource(nodeInfo, fsNodeClient, range, storageService, storageCredentials)
+            return FsNodeResource(nodeInfo, blockNodeService, range, storageService, storageCredentials)
         }
         if (clusterProperties.role == ClusterNodeType.EDGE &&
             clusterProperties.architecture != ClusterArchitecture.COMMIT_EDGE) {

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt

@@ -29,30 +29,28 @@ package com.tencent.bkrepo.common.artifact.manager.resource
 
 import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
 import com.tencent.bkrepo.common.artifact.stream.Range
+import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService
 import com.tencent.bkrepo.common.storage.core.StorageService
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
-import com.tencent.bkrepo.fs.server.api.FsNodeClient
+import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.node.NodeInfo
 
 /**
  * 文件节点
  * */
 class FsNodeResource(
     private val node: NodeInfo,
-    private val fsNodeClient: FsNodeClient,
+    private val blockNodeService: BlockNodeService,
     private val range: Range,
     private val storageService: StorageService,
     private val storageCredentials: StorageCredentials?
 ) : AbstractNodeResource() {
     override fun getArtifactInputStream(): ArtifactInputStream? {
         with(node) {
-            val blocks = fsNodeClient.listBlockResources(
-                projectId = projectId,
-                repoName = repoName,
-                path = fullPath,
-                startPos = range.start,
-                endPos = range.end
-            ).data ?: emptyList()
+            val blocks = blockNodeService.info(
+                nodeDetail = NodeDetail(this),
+                range = range
+            )
             return storageService.load(blocks, range, storageCredentials)
         }
     }

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecord.kt

@@ -63,6 +63,8 @@ data class ArtifactTransferRecord(
     val fullPath: String,
     @Column(name = "agent")
     val agent: String,
+    @Column(name = "userId")
+    val userId: String,
 ) {
     companion object {
         const val RECEIVE = "RECEIVE"

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/ArtifactTransferRecordLog.kt

@@ -46,6 +46,7 @@ class ArtifactTransferRecordLog(
     val repoName: String = record.repoName
     val agent: String = record.agent
     val fullPath: String = record.fullPath
+    val userId: String = record.userId
     val service: String? = commonTag["service"]
     val instance: String? = commonTag["instance"]
     val host: String? = commonTag["host"]

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/metrics/export/ArtifactMetricsExporter.kt

@@ -74,6 +74,7 @@ class ArtifactMetricsExporter(
         labels[ArtifactTransferRecord::elapsed.name] = record.elapsed.toString()
         labels[ArtifactTransferRecord::type.name] = record.type
         labels[ArtifactTransferRecord::agent.name] = record.agent
+        labels[ArtifactTransferRecord::userId.name] = record.userId
         return labels
     }
 

src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingDocument.kt → src/backend/common/common-metadata/build.gradle.kts

@@ -1,7 +1,7 @@
 /*
  * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
  *
- * Copyright (C) 2020 THL A29 Limited, a Tencent company.  All rights reserved.
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
  *
  * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
  *
@@ -28,17 +28,3 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-
-package com.tencent.bkrepo.common.mongo.reactive.dao
-
-/**
- * 分表Document
- */
-@Retention(AnnotationRetention.RUNTIME)
-@Target(AnnotationTarget.CLASS)
-annotation class ShardingDocument(
-    /**
-     * collection name
-     */
-    val collection: String = ""
-)