feat: 支持推送多架构镜像到oci仓库 #1689

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt

@@ -37,6 +37,9 @@ const val DOCKER_API_VERSION = "registry/2.0"
 const val DOCKER_CONTENT_DIGEST = "Docker-Content-Digest"
 const val DOCKER_UPLOAD_UUID = "Docker-Upload-Uuid"
 const val BLOB_UPLOAD_SESSION_ID = "Blob-Upload-Session-ID"
+const val ARCHITECTURE = "architecture"
+const val OS = "os"
+const val VARIANT = "variant"
 
 const val HTTP_FORWARDED_PROTO = "X-Forwarded-Proto"
 const val HTTP_PROTOCOL_HTTP = "http"
@@ -62,6 +65,7 @@ const val DIGEST = "oci_digest"
 const val SIZE = "size"
 const val SCHEMA_VERSION = "schemaVersion"
 const val IMAGE_VERSION = "blob_version"
+const val OLD_DOCKER_VERSION = "docker.manifest"
 const val MANIFEST_DIGEST = "manifest_digest"
 const val DIGEST_LIST = "digest_list"
 const val FORCE = "force"
@@ -101,12 +105,16 @@ const val EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca
 
 
 const val OCI_MANIFEST = "manifest.json"
+const val OCI_MANIFEST_LIST = "list.manifest.json"
 const val STAGE_TAG = "stageTag"
 
 const val TAG_LIST_REQUEST = "tagList"
 const val CATALOG_REQUEST = "catalog"
 const val REQUEST_IMAGE = "image"
 
+const val DOCKER_REPO_NAME = "docker.repoName"
+const val DOCKER_MANIFEST_DIGEST = "docker.manfiest.digest"
+
 // OCIScheme is the URL scheme for OCI-based requests
 const val OCI_SCHEME = "oci"
 
@@ -129,6 +137,8 @@ const val OCI_IMAGE_MANIFEST_MEDIA_TYPE = "application/vnd.oci.image.manifest.v1
 
 const val DOCKER_DISTRIBUTION_MANIFEST_V2 = "application/vnd.docker.distribution.manifest.v2+json"
 
+const val DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 = "application/vnd.docker.distribution.manifest.list.v2+json"
+
 // Content Descriptor
 const val CONTENT_DESCRIPTOR_MEDIA_TYPE = "application/vnd.oci.descriptor.v1+json"
 

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt

@@ -0,0 +1,16 @@
+package com.tencent.bkrepo.oci.model
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+data class ConfigSchema2(
+    var architecture: String,
+    var history: List<History>,
+    var os: String,
+    val variant: String? = null
+)
+
+data class History(
+    var created: String,
+    @JsonProperty("created_by")
+    var createdBy: String? = null
+)

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt

@@ -0,0 +1,8 @@
+package com.tencent.bkrepo.oci.model
+
+data class ManifestDescriptor(
+    override var mediaType: String,
+    override var size: Long,
+    override var digest: String,
+    var platform: Map<String, Any> = emptyMap()
+) : Descriptor(mediaType, size, digest)

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestList.kt

@@ -0,0 +1,7 @@
+package com.tencent.bkrepo.oci.model
+
+class ManifestList(
+    override var schemaVersion: Int,
+    var mediaType: String,
+    var manifests: List<ManifestDescriptor>,
+) : SchemaVersion(schemaVersion)

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt

@@ -27,6 +27,8 @@
 
 package com.tencent.bkrepo.oci.pojo.artifact
 
+import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_LIST_V2
+import com.tencent.bkrepo.oci.constant.IMAGE_INDEX_MEDIA_TYPE
 import com.tencent.bkrepo.oci.util.OciLocationUtils
 
 class OciManifestArtifactInfo(
@@ -35,17 +37,28 @@ class OciManifestArtifactInfo(
     packageName: String,
     version: String,
     val reference: String,
-    val isValidDigest: Boolean
+    val isValidDigest: Boolean,
+    var mediaType: String? = null
 ) : OciArtifactInfo(projectId, repoName, packageName, version) {
 
     override fun getArtifactFullPath(): String {
         return if(getArtifactMappingUri().isNullOrEmpty()) {
             if (isValidDigest) {
-                OciLocationUtils.buildDigestManifestPathWithReference(packageName, reference)
+                OciLocationUtils.buildDigestManifestPathWithReference(packageName, reference, isFatManifest())
+            } else if (isFatManifest()) {
+                OciLocationUtils.buildManifestListPath(packageName, reference)
             } else {
                 OciLocationUtils.buildManifestPath(packageName, reference)
             }
         } else getArtifactMappingUri()!!
+    }
+
+    fun isFatManifest() = FAT_MANIFEST_MEDIA_TYPES.contains(mediaType)
 
+    companion object {
+        private val FAT_MANIFEST_MEDIA_TYPES = listOf(
+            DOCKER_DISTRIBUTION_MANIFEST_LIST_V2,
+            IMAGE_INDEX_MEDIA_TYPE
+        )
     }
 }

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt

@@ -58,5 +58,7 @@ data class BasicInfo(
     @ApiModelProperty("修改者")
     val lastModifiedBy: String,
     @ApiModelProperty("修改时间")
-    val lastModifiedDate: String
+    val lastModifiedDate: String,
+    @ApiModelProperty("操作系统和架构信息")
+    val platform: List<String>
 )

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt

@@ -40,6 +40,8 @@ data class PackageVersionInfo(
     @ApiModelProperty("基础信息")
     val basic: BasicInfo,
     @ApiModelProperty("元数据信息")
-    val metadata: List<MetadataModel>
+    val metadata: List<MetadataModel>,
+    @ApiModelProperty("history")
+    val history: List<Any>
 )
 

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt

@@ -28,6 +28,7 @@
 package com.tencent.bkrepo.oci.util
 
 import com.tencent.bkrepo.common.api.constant.StringPool
+import com.tencent.bkrepo.oci.constant.OCI_MANIFEST_LIST
 import com.tencent.bkrepo.oci.constant.OCI_MANIFEST
 import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo
 import com.tencent.bkrepo.oci.pojo.digest.OciDigest
@@ -38,6 +39,10 @@ object OciLocationUtils {
         return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST
     }
 
+    fun buildManifestListPath(packageName: String, tag: String): String {
+        return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST_LIST
+    }
+
     fun buildManifestVersionFolderPath(packageName: String, tag: String): String {
         return buildManifestFolderPath(packageName) +"$tag/"
     }
@@ -46,8 +51,9 @@ object OciLocationUtils {
         return "/$packageName/manifest/"
     }
 
-    fun buildDigestManifestPathWithReference(packageName: String, reference: String): String {
-        return buildDigestManifestPath(packageName, OciDigest(reference))
+    fun buildDigestManifestPathWithReference(packageName: String, reference: String, isFatManifest: Boolean): String {
+        return buildManifestVersionFolderPath(packageName, reference) +
+                if (isFatManifest) OCI_MANIFEST_LIST else OCI_MANIFEST
     }
 
     private fun buildDigestManifestPath(packageName: String, ref: OciDigest): String {

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt

@@ -36,6 +36,7 @@ import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1
 import com.tencent.bkrepo.oci.constant.OciMessageCode
 import com.tencent.bkrepo.oci.exception.OciBadRequestException
 import com.tencent.bkrepo.oci.model.Descriptor
+import com.tencent.bkrepo.oci.model.ManifestList
 import com.tencent.bkrepo.oci.model.ManifestSchema1
 import com.tencent.bkrepo.oci.model.ManifestSchema2
 import com.tencent.bkrepo.oci.model.SchemaVersion
@@ -88,6 +89,14 @@ object OciUtils {
         }
     }
 
+    fun stringToManifestList(content: String): ManifestList {
+        try {
+            return content.readJsonString()
+        } catch (e: Exception) {
+            throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_INVALID, Strings.EMPTY)
+        }
+    }
+
     fun manifestIterator(manifest: ManifestSchema2): List<Descriptor> {
         val list = mutableListOf<Descriptor>()
         list.add(manifest.config)

src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt

@@ -57,6 +57,7 @@ import com.tencent.bkrepo.oci.constant.MEDIA_TYPE
 import com.tencent.bkrepo.oci.constant.N
 import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE
 import com.tencent.bkrepo.oci.constant.OLD_DOCKER_MEDIA_TYPE
+import com.tencent.bkrepo.oci.constant.OLD_DOCKER_VERSION
 import com.tencent.bkrepo.oci.constant.OciMessageCode
 import com.tencent.bkrepo.oci.constant.PATCH
 import com.tencent.bkrepo.oci.constant.POST
@@ -335,7 +336,9 @@ class OciRegistryLocalRepository(
         if (context.request.method == HttpMethod.HEAD.name) {
             return null
         }
-        val version = artifactResource.node?.metadata?.get(IMAGE_VERSION)?.toString() ?: return null
+        val version = artifactResource.node?.metadata?.get(IMAGE_VERSION)?.toString() ?: run {
+            artifactResource.node?.metadata?.get(OLD_DOCKER_VERSION)?.toString() ?: return null
+        }
         return PackageDownloadRecord(
             projectId = context.projectId,
             repoName = context.repoName,
@@ -374,7 +377,7 @@ class OciRegistryLocalRepository(
         }
 
         logger.info(
-            "The mediaType of Artifact $fullPath is $mediaType and it's contentType is $contentType" +
+            "The mediaType of Artifact ${node.fullPath} is $mediaType and it's contentType is $contentType" +
                 "in repo: ${context.artifactInfo.getRepoIdentify()}"
         )
         OciResponseUtils.buildDownloadResponse(
@@ -400,7 +403,15 @@ class OciRegistryLocalRepository(
             val oldDockerPath = ociOperationService.getDockerNode(artifactInfo)
                 ?: return null
             artifactInfo.setArtifactMappingUri(oldDockerPath)
-            ArtifactContextHolder.getNodeDetail(artifactInfo)
+            ArtifactContextHolder.getNodeDetail(artifactInfo) ?: run {
+                if (artifactInfo !is OciManifestArtifactInfo) return null
+                // 兼容 list.manifest.json
+                val manifestListPath = OciLocationUtils.buildManifestListPath(
+                    artifactInfo.packageName, artifactInfo.reference
+                )
+                artifactInfo.setArtifactMappingUri(manifestListPath)
+                ArtifactContextHolder.getNodeDetail(artifactInfo)
+            }
         }
     }
 
@@ -446,7 +457,9 @@ class OciRegistryLocalRepository(
         with(context) {
             val artifactInfo = context.artifactInfo as OciArtifactInfo
             val packageKey = PackageKeys.ofName(repo.type, artifactInfo.packageName)
-            val version = node.metadata[IMAGE_VERSION]?.toString() ?: return null
+            val version = node.metadata[IMAGE_VERSION]?.toString() ?: run {
+                node.metadata[OLD_DOCKER_VERSION]?.toString() ?: return null
+            }
             return packageClient.findVersionByName(projectId, repoName, packageKey, version).data
         }
     }

src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt

@@ -430,13 +430,17 @@ class OciRegistryRemoteRepository(
      */
     override fun onDownloadResponse(context: ArtifactDownloadContext, response: Response): ArtifactResource {
         logger.info("Remote download response will be processed")
+        val ociArtifactInfo = context.artifactInfo
+        if (ociArtifactInfo is OciManifestArtifactInfo) {
+            ociArtifactInfo.mediaType = response.header(HttpHeaders.CONTENT_TYPE)
+        }
         val artifactFile = createTempFile(response.body!!)
         val size = artifactFile.getSize()
         val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size))
         val node = cacheArtifact(context, artifactFile)
         val artifactResource = ArtifactResource(
             inputStream = artifactStream,
-            artifactName = context.artifactInfo.getResponseName(),
+            artifactName = ociArtifactInfo.getResponseName(),
             node = node,
             channel = ArtifactChannel.PROXY
         )

src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt

@@ -31,6 +31,7 @@
 
 package com.tencent.bkrepo.oci.artifact.resolver
 
+import com.tencent.bkrepo.common.api.constant.HttpHeaders
 import com.tencent.bkrepo.common.api.util.Preconditions
 import com.tencent.bkrepo.common.artifact.api.ArtifactInfo
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
@@ -69,7 +70,10 @@ class OciManifestArtifactInfoResolver : ArtifactInfoResolver {
                 val reference = attributes["reference"].toString().trim()
                 validate(packageName)
                 val isValidDigest = OciDigest.isValid(reference)
-                OciManifestArtifactInfo(projectId, repoName, packageName, "", reference, isValidDigest)
+                val contentType = request.getHeader(HttpHeaders.CONTENT_TYPE)
+                OciManifestArtifactInfo(
+                    projectId, repoName, packageName, "", reference, isValidDigest, contentType
+                )
             }
         }
     }

src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt

@@ -44,6 +44,8 @@ import com.tencent.bkrepo.common.service.util.HttpContextHolder
 import com.tencent.bkrepo.common.service.util.LocaleMessageUtils
 import com.tencent.bkrepo.oci.artifact.auth.OciLoginAuthHandler
 import com.tencent.bkrepo.oci.config.OciProperties
+import com.tencent.bkrepo.oci.constant.DOCKER_API_VERSION
+import com.tencent.bkrepo.oci.constant.DOCKER_HEADER_API_VERSION
 import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_CODE
 import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_DESCRIPTION
 import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_MESSAGE
@@ -72,6 +74,7 @@ class OciExceptionHandler(
     fun handleException(exception: AuthenticationException) {
         val response = HttpContextHolder.getResponse()
         response.contentType = MediaTypes.APPLICATION_JSON
+        response.setHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION)
         response.addHeader(
             HttpHeaders.WWW_AUTHENTICATE,
             OciLoginAuthHandler.AUTH_CHALLENGE_SERVICE_SCOPE.format(

src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt

@@ -73,9 +73,13 @@ class OciBlobServiceImpl(
 
     override fun startUploadBlob(artifactInfo: OciBlobArtifactInfo, artifactFile: ArtifactFile) {
         with(artifactInfo) {
-            logger.info("Handling bolb upload request $artifactInfo in ${getRepoIdentify()} .")
+            logger.info(
+                "Handling bolb upload request ${artifactInfo.digest}|${artifactInfo.uuid}|" +
+                    "${artifactInfo.mount}|${artifactInfo.from} in ${getRepoIdentify()}."
+            )
             if (digest.isNullOrBlank()) {
                 logger.info("Will use post then put to upload blob...")
+                // docker manifest mount upload blob
                 obtainSessionIdForUpload(artifactInfo)
             } else {
                 logger.info("Will use single post to upload blob...")