From f60be2f1b5fc6d7d332ccc8c53c164cc1259bb73 Mon Sep 17 00:00:00 2001 From: xujian Date: Fri, 20 Oct 2023 16:07:07 +0800 Subject: [PATCH 1/8] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0docker=20manifest?= =?UTF-8?q?,=20=E7=89=88=E6=9C=AC=E8=AF=A6=E6=83=85=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=20#1689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/constant/OciConstants.kt | 6 + .../tencent/bkrepo/oci/model/ConfigSchema2.kt | 15 ++ .../bkrepo/oci/model/ManifestDescriptor.kt | 8 ++ .../tencent/bkrepo/oci/model/ManifestList.kt | 7 + .../pojo/artifact/OciManifestArtifactInfo.kt | 19 ++- .../tencent/bkrepo/oci/pojo/user/BasicInfo.kt | 4 +- .../oci/pojo/user/PackageVersionInfo.kt | 4 +- .../bkrepo/oci/util/OciLocationUtils.kt | 5 + .../com/tencent/bkrepo/oci/util/OciUtils.kt | 9 ++ .../repository/OciRegistryLocalRepository.kt | 12 +- .../oci/exception/OciExceptionHandler.kt | 3 + .../oci/service/impl/OciBlobServiceImpl.kt | 6 +- .../service/impl/OciOperationServiceImpl.kt | 131 ++++++++++++++---- .../bkrepo/oci/util/ObjectBuildUtils.kt | 5 +- 14 files changed, 201 insertions(+), 33 deletions(-) create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestList.kt diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index c4e56a21fe..95336c6716 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -37,6 +37,9 @@ const val DOCKER_API_VERSION = "registry/2.0" const val DOCKER_CONTENT_DIGEST = "Docker-Content-Digest" const val DOCKER_UPLOAD_UUID = "Docker-Upload-Uuid" const val BLOB_UPLOAD_SESSION_ID = "Blob-Upload-Session-ID" +const val ARCHITECTURE = "architecture" +const val OS = "os" +const val VARIANT = "variant" const val HTTP_FORWARDED_PROTO = "X-Forwarded-Proto" const val HTTP_PROTOCOL_HTTP = "http" @@ -101,6 +104,7 @@ const val EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca const val OCI_MANIFEST = "manifest.json" +const val OCI_MANIFEST_LIST = "list.manifest.json" const val STAGE_TAG = "stageTag" const val TAG_LIST_REQUEST = "tagList" @@ -129,6 +133,8 @@ const val OCI_IMAGE_MANIFEST_MEDIA_TYPE = "application/vnd.oci.image.manifest.v1 const val DOCKER_DISTRIBUTION_MANIFEST_V2 = "application/vnd.docker.distribution.manifest.v2+json" +const val DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 = "application/vnd.docker.distribution.manifest.list.v2+json" + // Content Descriptor const val CONTENT_DESCRIPTOR_MEDIA_TYPE = "application/vnd.oci.descriptor.v1+json" diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt new file mode 100644 index 0000000000..63ece3c845 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt @@ -0,0 +1,15 @@ +package com.tencent.bkrepo.oci.model + +import com.fasterxml.jackson.annotation.JsonProperty + +data class ConfigSchema2( + var architecture: String, + var history: List, + var os: String +) + +data class History( + var created: String, + @JsonProperty("created_by") + var createdBy: String? = null +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt new file mode 100644 index 0000000000..c5afc8fd87 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt @@ -0,0 +1,8 @@ +package com.tencent.bkrepo.oci.model + +data class ManifestDescriptor( + override var mediaType: String, + override var size: Long, + override var digest: String, + var platform: Map = emptyMap() +):Descriptor(mediaType, size, digest) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestList.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestList.kt new file mode 100644 index 0000000000..6d4a46044c --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestList.kt @@ -0,0 +1,7 @@ +package com.tencent.bkrepo.oci.model + +class ManifestList( + override var schemaVersion: Int, + var mediaType: String, + var manifests: List, +) : SchemaVersion(schemaVersion) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt index 0314c8a54e..508d5fe3b4 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt @@ -27,6 +27,10 @@ package com.tencent.bkrepo.oci.pojo.artifact +import com.tencent.bkrepo.common.api.constant.HttpHeaders +import com.tencent.bkrepo.common.service.util.HeaderUtils +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 import com.tencent.bkrepo.oci.util.OciLocationUtils class OciManifestArtifactInfo( @@ -41,9 +45,20 @@ class OciManifestArtifactInfo( override fun getArtifactFullPath(): String { return if(getArtifactMappingUri().isNullOrEmpty()) { if (isValidDigest) { - OciLocationUtils.buildDigestManifestPathWithReference(packageName, reference) + // PUT 请求时取 digest 做为tag名 + if (HttpContextHolder.getRequest().method.equals("PUT", ignoreCase = true)) { + OciLocationUtils.buildManifestPath(packageName, reference) + } else { + OciLocationUtils.buildDigestManifestPathWithReference(packageName, reference) + } } else { - OciLocationUtils.buildManifestPath(packageName, reference) + // 根据 Content-type 区分 manifest.json / list.manifest.json + val mediaType = HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) + if (mediaType == DOCKER_DISTRIBUTION_MANIFEST_LIST_V2) { + OciLocationUtils.buildManifestListPath(packageName, reference) + } else { + OciLocationUtils.buildManifestPath(packageName, reference) + } } } else getArtifactMappingUri()!! diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt index cc4849fca2..de1f387c3f 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt @@ -58,5 +58,7 @@ data class BasicInfo( @ApiModelProperty("修改者") val lastModifiedBy: String, @ApiModelProperty("修改时间") - val lastModifiedDate: String + val lastModifiedDate: String, + @ApiModelProperty("操作系统和架构信息") + val os: List ) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt index 698937409f..f07319fc59 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/PackageVersionInfo.kt @@ -40,6 +40,8 @@ data class PackageVersionInfo( @ApiModelProperty("基础信息") val basic: BasicInfo, @ApiModelProperty("元数据信息") - val metadata: List + val metadata: List, + @ApiModelProperty("history") + val history: List ) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index c7b1dbb7c2..26cb44d8d0 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.oci.util import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.oci.constant.OCI_MANIFEST_LIST import com.tencent.bkrepo.oci.constant.OCI_MANIFEST import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest @@ -38,6 +39,10 @@ object OciLocationUtils { return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST } + fun buildManifestListPath(packageName: String, tag: String): String { + return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST_LIST + } + fun buildManifestVersionFolderPath(packageName: String, tag: String): String { return buildManifestFolderPath(packageName) +"$tag/" } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt index 29004f80ff..39f7315312 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.model.Descriptor +import com.tencent.bkrepo.oci.model.ManifestList import com.tencent.bkrepo.oci.model.ManifestSchema1 import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.SchemaVersion @@ -88,6 +89,14 @@ object OciUtils { } } + fun stringToManifestList(content: String): ManifestList { + try { + return content.readJsonString() + } catch (e: Exception) { + throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_INVALID, Strings.EMPTY) + } + } + fun manifestIterator(manifest: ManifestSchema2): List { val list = mutableListOf() list.add(manifest.config) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index cb8212f64c..15c0b34166 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -374,7 +374,7 @@ class OciRegistryLocalRepository( } logger.info( - "The mediaType of Artifact $fullPath is $mediaType and it's contentType is $contentType" + + "The mediaType of Artifact ${node.fullPath} is $mediaType and it's contentType is $contentType" + "in repo: ${context.artifactInfo.getRepoIdentify()}" ) OciResponseUtils.buildDownloadResponse( @@ -400,7 +400,15 @@ class OciRegistryLocalRepository( val oldDockerPath = ociOperationService.getDockerNode(artifactInfo) ?: return null artifactInfo.setArtifactMappingUri(oldDockerPath) - ArtifactContextHolder.getNodeDetail(artifactInfo) + ArtifactContextHolder.getNodeDetail(artifactInfo) ?: run { + if (artifactInfo !is OciManifestArtifactInfo) return null + // 兼容 list.manifest.json + val manifestListPath = OciLocationUtils.buildManifestListPath( + artifactInfo.packageName, artifactInfo.reference + ) + artifactInfo.setArtifactMappingUri(manifestListPath) + ArtifactContextHolder.getNodeDetail(artifactInfo) + } } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt index 88aac6b016..a73602edc6 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/exception/OciExceptionHandler.kt @@ -44,6 +44,8 @@ import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.service.util.LocaleMessageUtils import com.tencent.bkrepo.oci.artifact.auth.OciLoginAuthHandler import com.tencent.bkrepo.oci.config.OciProperties +import com.tencent.bkrepo.oci.constant.DOCKER_API_VERSION +import com.tencent.bkrepo.oci.constant.DOCKER_HEADER_API_VERSION import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_CODE import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_DESCRIPTION import com.tencent.bkrepo.oci.constant.UNAUTHORIZED_MESSAGE @@ -72,6 +74,7 @@ class OciExceptionHandler( fun handleException(exception: AuthenticationException) { val response = HttpContextHolder.getResponse() response.contentType = MediaTypes.APPLICATION_JSON + response.setHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION) response.addHeader( HttpHeaders.WWW_AUTHENTICATE, OciLoginAuthHandler.AUTH_CHALLENGE_SERVICE_SCOPE.format( diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index 53646481db..9d8f19182e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -73,9 +73,13 @@ class OciBlobServiceImpl( override fun startUploadBlob(artifactInfo: OciBlobArtifactInfo, artifactFile: ArtifactFile) { with(artifactInfo) { - logger.info("Handling bolb upload request $artifactInfo in ${getRepoIdentify()} .") + logger.info( + "Handling bolb upload request ${artifactInfo.digest}|${artifactInfo.uuid}|" + + "${artifactInfo.mount}|${artifactInfo.from} in ${getRepoIdentify()}." + ) if (digest.isNullOrBlank()) { logger.info("Will use post then put to upload blob...") + // docker manifest mount upload blob obtainSessionIdForUpload(artifactInfo) } else { logger.info("Will use single post to upload blob...") diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index a2379d8bdd..ca4441274c 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -30,6 +30,7 @@ package com.tencent.bkrepo.oci.service.impl import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE @@ -53,6 +54,7 @@ import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties +import com.tencent.bkrepo.oci.constant.ARCHITECTURE import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE @@ -64,9 +66,11 @@ import com.tencent.bkrepo.oci.constant.MANIFEST_DIGEST import com.tencent.bkrepo.oci.constant.MD5 import com.tencent.bkrepo.oci.constant.NODE_FULL_PATH import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE +import com.tencent.bkrepo.oci.constant.OCI_MANIFEST_LIST import com.tencent.bkrepo.oci.constant.OCI_NODE_FULL_PATH import com.tencent.bkrepo.oci.constant.OCI_NODE_SIZE import com.tencent.bkrepo.oci.constant.OCI_PACKAGE_NAME +import com.tencent.bkrepo.oci.constant.OS import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PROXY_URL import com.tencent.bkrepo.oci.constant.REPO_TYPE @@ -76,7 +80,10 @@ import com.tencent.bkrepo.oci.exception.OciFileNotFoundException import com.tencent.bkrepo.oci.exception.OciVersionNotFoundException import com.tencent.bkrepo.oci.extension.ImagePackageInfoPullExtension import com.tencent.bkrepo.oci.extension.ImagePackagePullContext +import com.tencent.bkrepo.oci.model.ConfigSchema2 import com.tencent.bkrepo.oci.model.Descriptor +import com.tencent.bkrepo.oci.model.History +import com.tencent.bkrepo.oci.model.ManifestList import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.TOciReplicationRecord import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo @@ -298,11 +305,50 @@ class OciOperationServiceImpl( OciMessageCode.OCI_VERSION_NOT_FOUND, "$packageKey/$version", "$projectId|$repoName" ) val packageVersion = packageClient.findVersionByName(projectId, repoName, packageKey, version).data!! - val basicInfo = ObjectBuildUtils.buildBasicInfo(nodeDetail, packageVersion) - return PackageVersionInfo(basicInfo, packageVersion.packageMetadata) + val pair = getManifestInfo(nodeDetail, repoDetail, name) + val basicInfo = ObjectBuildUtils.buildBasicInfo(nodeDetail, packageVersion, pair.first) + return PackageVersionInfo(basicInfo, packageVersion.packageMetadata, pair.second) } } + private fun getManifestInfo( + nodeDetail: NodeDetail, + repoDetail: RepositoryDetail, + packageName: String + ): Pair, List> { + val os = mutableListOf() + var history = listOf() + if (nodeDetail.name == OCI_MANIFEST_LIST) { + // manifestList + loadManifestList( + nodeDetail.sha256!!, + nodeDetail.size, + repoDetail.storageCredentials + )?.let { manifestList -> + manifestList.manifests.forEach { manifest -> + val map = manifest.platform + os.add("${map[OS]}/${map[ARCHITECTURE]}") + } + } + } else { + val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, repoDetail.storageCredentials) + // config + manifest?.config?.digest?.let { configDigest -> + val configNode = getImageNodeDetail( + nodeDetail.projectId, + nodeDetail.repoName, + packageName, + configDigest + ) + val inputStream = storageManager.loadArtifactInputStream(configNode, repoDetail.storageCredentials) + val config = JsonUtils.objectMapper.readValue(inputStream, ConfigSchema2::class.java) + os.add("${config.os}/${config.architecture}") + history = config.history + } + } + return Pair(os, history) + } + /** * 获取node节点 * 查不到抛出OciFileNotFoundException异常 @@ -322,7 +368,7 @@ class OciOperationServiceImpl( packageName = name, reference = version, isValidDigest = false, - version = StringPool.EMPTY + version = version ) } else { // 返回blob文件的节点信息 @@ -338,10 +384,15 @@ class OciOperationServiceImpl( val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: run { val oldDockerFullPath = getDockerNode(ociArtifactInfo) ?: return@run null nodeClient.getNodeDetail(projectId, repoName, oldDockerFullPath).data ?: run { - logger.warn("node [$fullPath] don't found.") - null + if (ociArtifactInfo !is OciManifestArtifactInfo) return null + // 兼容 list.manifest.json + val manifestListPath = OciLocationUtils.buildManifestListPath( + ociArtifactInfo.packageName, ociArtifactInfo.version + ) + nodeClient.getNodeDetail(projectId, repoName, manifestListPath).data } } + logger.info("Get images node fullPath:$fullPath, nodeDetail fullPath:${nodeDetail?.fullPath}") return nodeDetail } @@ -429,36 +480,48 @@ class OciOperationServiceImpl( // https://github.com/docker/docker-ce/blob/master/components/engine/distribution/push_v2.go // docker 客户端上传manifest时先按照schema2的格式上传， // 如失败则按照schema1格式上传，但是非docker客户端不兼容schema1版本manifest - val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) - ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) - // 更新manifest文件的metadata - val mediaType = if (manifest.mediaType.isNullOrEmpty()) { - HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE + val mediaType: String + var digestList = listOf() + // 需要区分 manifest.json 和 list.manifest.json + if (nodeDetail.name == OCI_MANIFEST_LIST) { + val manifestList = loadManifestList(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) + mediaType = manifestList!!.mediaType + doPackageOperations( + manifestPath = nodeDetail.fullPath, + ociArtifactInfo = ociArtifactInfo, + manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), + size = nodeDetail.size, + sourceType = sourceType, + userId = SecurityUtils.getUserId() + ) } else { - manifest.mediaType + val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) + ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) + mediaType = if (manifest.mediaType.isNullOrEmpty()) { + HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE + } else { + manifest.mediaType!! + } + digestList = OciUtils.manifestIteratorDigest(manifest) + if (ociArtifactInfo.packageName.isEmpty()) return + // 处理manifest中的blob数据 + syncBlobInfo( + ociArtifactInfo = ociArtifactInfo, + manifest = manifest, + nodeDetail = nodeDetail, + sourceType = sourceType + ) } - val digestList = OciUtils.manifestIteratorDigest(manifest) - // 更新manifest节点元数据 updateNodeMetaData( projectId = ociArtifactInfo.projectId, repoName = ociArtifactInfo.repoName, version = ociArtifactInfo.reference, fullPath = nodeDetail.fullPath, - mediaType = mediaType!!, + mediaType = mediaType, digestList = digestList, sourceType = sourceType ) - - - if (ociArtifactInfo.packageName.isEmpty()) return - // 处理manifest中的blob数据 - syncBlobInfo( - ociArtifactInfo = ociArtifactInfo, - manifest = manifest, - nodeDetail = nodeDetail, - sourceType = sourceType - ) } @@ -476,6 +539,26 @@ class OciOperationServiceImpl( OciUtils.stringToManifestV2(manifestBytes) } catch (e: Exception) { + logger.error("load manifest error, sha256:$sha256") + null + } + } + + private fun loadManifestList( + sha256: String, + size: Long, + storageCredentials: StorageCredentials? + ): ManifestList? { + return try { + val manifestBytes = storageService.load( + sha256, + Range.full(size), + storageCredentials + )!!.readText() + + OciUtils.stringToManifestList(manifestBytes) + } catch (e: Exception) { + logger.error("load manifest list error, sha256:$sha256") null } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt index c7a3c534d8..6b1754acd5 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt @@ -212,7 +212,7 @@ object ObjectBuildUtils { ) } - fun buildBasicInfo(nodeDetail: NodeDetail, packageVersion: PackageVersion): BasicInfo { + fun buildBasicInfo(nodeDetail: NodeDetail, packageVersion: PackageVersion, os: List): BasicInfo { with(nodeDetail) { return BasicInfo( version = packageVersion.name, @@ -227,7 +227,8 @@ object ObjectBuildUtils { createdBy = createdBy, createdDate = createdDate, lastModifiedBy = lastModifiedBy, - lastModifiedDate = lastModifiedDate + lastModifiedDate = lastModifiedDate, + os = os ) } } From b54cf5892c71b11d2f6679e41894938ecdfbcc41 Mon Sep 17 00:00:00 2001 From: xujian Date: Tue, 24 Oct 2023 15:10:42 +0800 Subject: [PATCH 2/8] =?UTF-8?q?feat:=20=E6=A0=B9=E6=8D=AEnode=20Metadata?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E7=89=88=E6=9C=AC=E4=BF=A1=E6=81=AF=E5=85=BC?= =?UTF-8?q?=E5=AE=B9=E6=97=A7docker=E5=88=B6=E5=93=81=20#1689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/oci/constant/OciConstants.kt | 1 + .../artifact/repository/OciRegistryLocalRepository.kt | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index 95336c6716..4188655870 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -65,6 +65,7 @@ const val DIGEST = "oci_digest" const val SIZE = "size" const val SCHEMA_VERSION = "schemaVersion" const val IMAGE_VERSION = "blob_version" +const val OLD_DOCKER_VERSION = "docker.manifest" const val MANIFEST_DIGEST = "manifest_digest" const val DIGEST_LIST = "digest_list" const val FORCE = "force" diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 15c0b34166..0c0be78e65 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -57,6 +57,7 @@ import com.tencent.bkrepo.oci.constant.MEDIA_TYPE import com.tencent.bkrepo.oci.constant.N import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE import com.tencent.bkrepo.oci.constant.OLD_DOCKER_MEDIA_TYPE +import com.tencent.bkrepo.oci.constant.OLD_DOCKER_VERSION import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PATCH import com.tencent.bkrepo.oci.constant.POST @@ -335,7 +336,9 @@ class OciRegistryLocalRepository( if (context.request.method == HttpMethod.HEAD.name) { return null } - val version = artifactResource.node?.metadata?.get(IMAGE_VERSION)?.toString() ?: return null + val version = artifactResource.node?.metadata?.get(IMAGE_VERSION)?.toString() ?: run { + artifactResource.node?.metadata?.get(OLD_DOCKER_VERSION)?.toString() ?: return null + } return PackageDownloadRecord( projectId = context.projectId, repoName = context.repoName, @@ -454,7 +457,9 @@ class OciRegistryLocalRepository( with(context) { val artifactInfo = context.artifactInfo as OciArtifactInfo val packageKey = PackageKeys.ofName(repo.type, artifactInfo.packageName) - val version = node.metadata[IMAGE_VERSION]?.toString() ?: return null + val version = node.metadata[IMAGE_VERSION]?.toString() ?: run { + node.metadata[OLD_DOCKER_VERSION]?.toString() ?: return null + } return packageClient.findVersionByName(projectId, repoName, packageKey, version).data } } From 0b2bb4633dd773345bd6d201026a1694c77ba445 Mon Sep 17 00:00:00 2001 From: xujian Date: Wed, 25 Oct 2023 14:06:53 +0800 Subject: [PATCH 3/8] =?UTF-8?q?feat:=20oci=E5=88=9B=E5=BB=BA=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E6=97=B6=E5=88=9B=E5=BB=BA=E9=BB=98=E8=AE=A4=E5=85=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=20#1689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/constant/OciConstants.kt | 3 ++ .../service/impl/OciOperationServiceImpl.kt | 30 ++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index 4188655870..f9cd952b0f 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -112,6 +112,9 @@ const val TAG_LIST_REQUEST = "tagList" const val CATALOG_REQUEST = "catalog" const val REQUEST_IMAGE = "image" +const val DOCKER_REPO_NAME = "docker.repoName" +const val DOCKER_MANIFEST_DIGEST = "docker.manfiest.digest" + // OCIScheme is the URL scheme for OCI-based requests const val OCI_SCHEME = "oci" diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index ca4441274c..35c9534dc0 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -59,6 +59,8 @@ import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE import com.tencent.bkrepo.oci.constant.DESCRIPTION +import com.tencent.bkrepo.oci.constant.DOCKER_MANIFEST_DIGEST +import com.tencent.bkrepo.oci.constant.DOCKER_REPO_NAME import com.tencent.bkrepo.oci.constant.DOWNLOADS import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_BY import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_DATE @@ -70,6 +72,8 @@ import com.tencent.bkrepo.oci.constant.OCI_MANIFEST_LIST import com.tencent.bkrepo.oci.constant.OCI_NODE_FULL_PATH import com.tencent.bkrepo.oci.constant.OCI_NODE_SIZE import com.tencent.bkrepo.oci.constant.OCI_PACKAGE_NAME +import com.tencent.bkrepo.oci.constant.OLD_DOCKER_MEDIA_TYPE +import com.tencent.bkrepo.oci.constant.OLD_DOCKER_VERSION import com.tencent.bkrepo.oci.constant.OS import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PROXY_URL @@ -102,6 +106,7 @@ import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciLocationUtils.buildBlobsFolderPath import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.oci.util.OciUtils +import com.tencent.bkrepo.replication.constant.SHA256 import com.tencent.bkrepo.repository.api.MetadataClient import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient @@ -486,12 +491,20 @@ class OciOperationServiceImpl( if (nodeDetail.name == OCI_MANIFEST_LIST) { val manifestList = loadManifestList(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) mediaType = manifestList!!.mediaType + val metadata = mutableMapOf( + OLD_DOCKER_VERSION to ociArtifactInfo.reference, + SHA256 to nodeDetail.sha256!!, + DOCKER_REPO_NAME to ociArtifactInfo.packageName, + DOCKER_MANIFEST_DIGEST to OciDigest.fromSha256(nodeDetail.sha256!!).toString(), + OLD_DOCKER_MEDIA_TYPE to mediaType, + ) doPackageOperations( manifestPath = nodeDetail.fullPath, ociArtifactInfo = ociArtifactInfo, manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), size = nodeDetail.size, sourceType = sourceType, + metadata = metadata, userId = SecurityUtils.getUserId() ) } else { @@ -655,6 +668,15 @@ class OciOperationServiceImpl( ) // 如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 if (existFlag) { + val mediaType = manifest.mediaType ?: HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) + ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE + val metadata = mutableMapOf( + OLD_DOCKER_VERSION to ociArtifactInfo.reference, + SHA256 to nodeDetail.sha256!!, + DOCKER_REPO_NAME to ociArtifactInfo.packageName, + DOCKER_MANIFEST_DIGEST to OciDigest.fromSha256(nodeDetail.sha256!!).toString(), + OLD_DOCKER_MEDIA_TYPE to mediaType + ) // 第三方同步的索引更新等所有文件全部上传完成后才去进行 // 根据flag生成package信息以及package version信息 doPackageOperations( @@ -663,6 +685,7 @@ class OciOperationServiceImpl( manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), size = size, sourceType = sourceType, + metadata = metadata, userId = userId ) return true @@ -791,6 +814,7 @@ class OciOperationServiceImpl( manifestDigest: OciDigest, size: Long, sourceType: ArtifactChannel? = null, + metadata: MutableMap, userId: String = SecurityUtils.getUserId() ) { with(ociArtifactInfo) { @@ -798,10 +822,8 @@ class OciOperationServiceImpl( // 针对支持多仓库类型，如docker和oci val repoType = repositoryClient.getRepoDetail(projectId, repoName).data!!.type.name val packageKey = PackageKeys.ofName(repoType.toLowerCase(), packageName) - val metadata = mutableMapOf(MANIFEST_DIGEST to manifestDigest.toString()) - .apply { - sourceType?.let { this[SOURCE_TYPE] = sourceType } - } + metadata[MANIFEST_DIGEST] = manifestDigest.toString() + sourceType?.let { metadata.put(SOURCE_TYPE, sourceType) } val request = ObjectBuildUtils.buildPackageVersionCreateRequest( ociArtifactInfo = this, packageName = packageName, From 0ecef9facee7ba0c284cdde0f4440b1e5b721fbc Mon Sep 17 00:00:00 2001 From: ericwu Date: Sat, 28 Oct 2023 15:41:07 +0800 Subject: [PATCH 4/8] =?UTF-8?q?fix:=20oci=20manifest=20list=E6=90=BA?= =?UTF-8?q?=E5=B8=A6os.features=E6=97=B6=E6=8E=A8=E9=80=81=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=20#1689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt index c5afc8fd87..8013ff995e 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ManifestDescriptor.kt @@ -4,5 +4,5 @@ data class ManifestDescriptor( override var mediaType: String, override var size: Long, override var digest: String, - var platform: Map = emptyMap() -):Descriptor(mediaType, size, digest) + var platform: Map = emptyMap() +) : Descriptor(mediaType, size, digest) From ff5d881641a209c98227d6fcfcfd5d92607bd79d Mon Sep 17 00:00:00 2001 From: ericwu Date: Tue, 23 Jan 2024 16:17:07 +0800 Subject: [PATCH 5/8] =?UTF-8?q?feat:=20oci=E9=95=9C=E5=83=8F=E6=9E=B6?= =?UTF-8?q?=E6=9E=84=E4=BF=A1=E6=81=AF=E6=96=B0=E5=A2=9Evariant=20#1689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/oci/model/ConfigSchema2.kt | 3 ++- .../com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt | 2 +- .../oci/service/impl/OciOperationServiceImpl.kt | 12 ++++++++---- .../com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt | 4 ++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt index 63ece3c845..538198e472 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/model/ConfigSchema2.kt @@ -5,7 +5,8 @@ import com.fasterxml.jackson.annotation.JsonProperty data class ConfigSchema2( var architecture: String, var history: List, - var os: String + var os: String, + val variant: String? = null ) data class History( diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt index de1f387c3f..19715bf5e1 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/user/BasicInfo.kt @@ -60,5 +60,5 @@ data class BasicInfo( @ApiModelProperty("修改时间") val lastModifiedDate: String, @ApiModelProperty("操作系统和架构信息") - val os: List + val platform: List ) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 35c9534dc0..4d0252346b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -30,6 +30,7 @@ package com.tencent.bkrepo.oci.service.impl import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.constant.ensurePrefix import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.artifact.api.ArtifactFile @@ -78,6 +79,7 @@ import com.tencent.bkrepo.oci.constant.OS import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.PROXY_URL import com.tencent.bkrepo.oci.constant.REPO_TYPE +import com.tencent.bkrepo.oci.constant.VARIANT import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.exception.OciFileNotFoundException @@ -321,7 +323,7 @@ class OciOperationServiceImpl( repoDetail: RepositoryDetail, packageName: String ): Pair, List> { - val os = mutableListOf() + val platform = mutableListOf() var history = listOf() if (nodeDetail.name == OCI_MANIFEST_LIST) { // manifestList @@ -332,7 +334,9 @@ class OciOperationServiceImpl( )?.let { manifestList -> manifestList.manifests.forEach { manifest -> val map = manifest.platform - os.add("${map[OS]}/${map[ARCHITECTURE]}") + platform.add( + "${map[OS]}/${map[ARCHITECTURE]}" + (map[VARIANT]?.toString()?.ensurePrefix("/") ?: "") + ) } } } else { @@ -347,11 +351,11 @@ class OciOperationServiceImpl( ) val inputStream = storageManager.loadArtifactInputStream(configNode, repoDetail.storageCredentials) val config = JsonUtils.objectMapper.readValue(inputStream, ConfigSchema2::class.java) - os.add("${config.os}/${config.architecture}") + platform.add("${config.os}/${config.architecture}" + (config.variant?.ensurePrefix("/") ?: "")) history = config.history } } - return Pair(os, history) + return Pair(platform, history) } /** diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt index 6b1754acd5..76618d3d3d 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt @@ -212,7 +212,7 @@ object ObjectBuildUtils { ) } - fun buildBasicInfo(nodeDetail: NodeDetail, packageVersion: PackageVersion, os: List): BasicInfo { + fun buildBasicInfo(nodeDetail: NodeDetail, packageVersion: PackageVersion, platform: List): BasicInfo { with(nodeDetail) { return BasicInfo( version = packageVersion.name, @@ -228,7 +228,7 @@ object ObjectBuildUtils { createdDate = createdDate, lastModifiedBy = lastModifiedBy, lastModifiedDate = lastModifiedDate, - os = os + platform = platform ) } } From e91d60a1fed452f264d208a9af96612ec54606c4 Mon Sep 17 00:00:00 2001 From: ericwu Date: Tue, 23 Jan 2024 21:12:26 +0800 Subject: [PATCH 6/8] =?UTF-8?q?feat:=20oci=20manifest=20list=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=20#1689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt index 508d5fe3b4..0f8a3a1ee4 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 +import com.tencent.bkrepo.oci.constant.IMAGE_INDEX_MEDIA_TYPE import com.tencent.bkrepo.oci.util.OciLocationUtils class OciManifestArtifactInfo( @@ -54,7 +55,7 @@ class OciManifestArtifactInfo( } else { // 根据 Content-type 区分 manifest.json / list.manifest.json val mediaType = HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) - if (mediaType == DOCKER_DISTRIBUTION_MANIFEST_LIST_V2) { + if (mediaType == DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 || mediaType == IMAGE_INDEX_MEDIA_TYPE) { OciLocationUtils.buildManifestListPath(packageName, reference) } else { OciLocationUtils.buildManifestPath(packageName, reference) From 135ce31e3e3404ae1910adce780dab1002bbc3ee Mon Sep 17 00:00:00 2001 From: xujian Date: Sun, 18 Feb 2024 16:03:38 +0800 Subject: [PATCH 7/8] =?UTF-8?q?perf:=20=E5=8F=96=E6=B6=88=E4=B8=8D?= =?UTF-8?q?=E5=BF=85=E8=A6=81=E7=9A=84=E5=8F=82=E6=95=B0=E8=B5=8B=E5=80=BC?= =?UTF-8?q?=20#1689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/oci/service/impl/OciOperationServiceImpl.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 4d0252346b..21d0830995 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -377,7 +377,7 @@ class OciOperationServiceImpl( packageName = name, reference = version, isValidDigest = false, - version = version + version = StringPool.EMPTY ) } else { // 返回blob文件的节点信息 @@ -396,7 +396,7 @@ class OciOperationServiceImpl( if (ociArtifactInfo !is OciManifestArtifactInfo) return null // 兼容 list.manifest.json val manifestListPath = OciLocationUtils.buildManifestListPath( - ociArtifactInfo.packageName, ociArtifactInfo.version + ociArtifactInfo.packageName, ociArtifactInfo.reference ) nodeClient.getNodeDetail(projectId, repoName, manifestListPath).data } From 5265eb3829f6bd8928eb951e6a424b4efd7212ef Mon Sep 17 00:00:00 2001 From: ericwu Date: Fri, 15 Mar 2024 15:37:20 +0800 Subject: [PATCH 8/8] =?UTF-8?q?refactor:=20oci=20manifest=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E4=BC=98=E5=8C=96=20#1689?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pojo/artifact/OciManifestArtifactInfo.kt | 31 +++++------ .../bkrepo/oci/util/OciLocationUtils.kt | 5 +- .../repository/OciRegistryRemoteRepository.kt | 6 ++- .../OciManifestArtifactInfoResolver.kt | 6 ++- .../service/impl/OciOperationServiceImpl.kt | 51 ++++++++++--------- 5 files changed, 53 insertions(+), 46 deletions(-) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt index 0f8a3a1ee4..7e787713df 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciManifestArtifactInfo.kt @@ -27,9 +27,6 @@ package com.tencent.bkrepo.oci.pojo.artifact -import com.tencent.bkrepo.common.api.constant.HttpHeaders -import com.tencent.bkrepo.common.service.util.HeaderUtils -import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 import com.tencent.bkrepo.oci.constant.IMAGE_INDEX_MEDIA_TYPE import com.tencent.bkrepo.oci.util.OciLocationUtils @@ -40,28 +37,28 @@ class OciManifestArtifactInfo( packageName: String, version: String, val reference: String, - val isValidDigest: Boolean + val isValidDigest: Boolean, + var mediaType: String? = null ) : OciArtifactInfo(projectId, repoName, packageName, version) { override fun getArtifactFullPath(): String { return if(getArtifactMappingUri().isNullOrEmpty()) { if (isValidDigest) { - // PUT 请求时取 digest 做为tag名 - if (HttpContextHolder.getRequest().method.equals("PUT", ignoreCase = true)) { - OciLocationUtils.buildManifestPath(packageName, reference) - } else { - OciLocationUtils.buildDigestManifestPathWithReference(packageName, reference) - } + OciLocationUtils.buildDigestManifestPathWithReference(packageName, reference, isFatManifest()) + } else if (isFatManifest()) { + OciLocationUtils.buildManifestListPath(packageName, reference) } else { - // 根据 Content-type 区分 manifest.json / list.manifest.json - val mediaType = HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) - if (mediaType == DOCKER_DISTRIBUTION_MANIFEST_LIST_V2 || mediaType == IMAGE_INDEX_MEDIA_TYPE) { - OciLocationUtils.buildManifestListPath(packageName, reference) - } else { - OciLocationUtils.buildManifestPath(packageName, reference) - } + OciLocationUtils.buildManifestPath(packageName, reference) } } else getArtifactMappingUri()!! + } + + fun isFatManifest() = FAT_MANIFEST_MEDIA_TYPES.contains(mediaType) + companion object { + private val FAT_MANIFEST_MEDIA_TYPES = listOf( + DOCKER_DISTRIBUTION_MANIFEST_LIST_V2, + IMAGE_INDEX_MEDIA_TYPE + ) } } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index 26cb44d8d0..92d84a3d54 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -51,8 +51,9 @@ object OciLocationUtils { return "/$packageName/manifest/" } - fun buildDigestManifestPathWithReference(packageName: String, reference: String): String { - return buildDigestManifestPath(packageName, OciDigest(reference)) + fun buildDigestManifestPathWithReference(packageName: String, reference: String, isFatManifest: Boolean): String { + return buildManifestVersionFolderPath(packageName, reference) + + if (isFatManifest) OCI_MANIFEST_LIST else OCI_MANIFEST } private fun buildDigestManifestPath(packageName: String, ref: OciDigest): String { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 0991892b88..0e227cf814 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -430,13 +430,17 @@ class OciRegistryRemoteRepository( */ override fun onDownloadResponse(context: ArtifactDownloadContext, response: Response): ArtifactResource { logger.info("Remote download response will be processed") + val ociArtifactInfo = context.artifactInfo + if (ociArtifactInfo is OciManifestArtifactInfo) { + ociArtifactInfo.mediaType = response.header(HttpHeaders.CONTENT_TYPE) + } val artifactFile = createTempFile(response.body!!) val size = artifactFile.getSize() val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size)) val node = cacheArtifact(context, artifactFile) val artifactResource = ArtifactResource( inputStream = artifactStream, - artifactName = context.artifactInfo.getResponseName(), + artifactName = ociArtifactInfo.getResponseName(), node = node, channel = ArtifactChannel.PROXY ) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt index 68dfb1894a..3bc4edfdc5 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciManifestArtifactInfoResolver.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.oci.artifact.resolver +import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder @@ -69,7 +70,10 @@ class OciManifestArtifactInfoResolver : ArtifactInfoResolver { val reference = attributes["reference"].toString().trim() validate(packageName) val isValidDigest = OciDigest.isValid(reference) - OciManifestArtifactInfo(projectId, repoName, packageName, "", reference, isValidDigest) + val contentType = request.getHeader(HttpHeaders.CONTENT_TYPE) + OciManifestArtifactInfo( + projectId, repoName, packageName, "", reference, isValidDigest, contentType + ) } } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index 21d0830995..db0c7f106a 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -28,7 +28,6 @@ package com.tencent.bkrepo.oci.service.impl import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER -import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.constant.ensurePrefix import com.tencent.bkrepo.common.api.util.JsonUtils @@ -50,7 +49,6 @@ import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo @@ -489,12 +487,18 @@ class OciOperationServiceImpl( // https://github.com/docker/docker-ce/blob/master/components/engine/distribution/push_v2.go // docker 客户端上传manifest时先按照schema2的格式上传， // 如失败则按照schema1格式上传，但是非docker客户端不兼容schema1版本manifest - val mediaType: String - var digestList = listOf() // 需要区分 manifest.json 和 list.manifest.json - if (nodeDetail.name == OCI_MANIFEST_LIST) { - val manifestList = loadManifestList(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) - mediaType = manifestList!!.mediaType + if (ociArtifactInfo.isFatManifest()) { + val mediaType = loadManifestList(nodeDetail.sha256!!, nodeDetail.size, storageCredentials)!!.mediaType + // 更新manifest节点元数据 + updateNodeMetaData( + projectId = ociArtifactInfo.projectId, + repoName = ociArtifactInfo.repoName, + version = ociArtifactInfo.reference, + fullPath = nodeDetail.fullPath, + mediaType = mediaType, + sourceType = sourceType + ) val metadata = mutableMapOf( OLD_DOCKER_VERSION to ociArtifactInfo.reference, SHA256 to nodeDetail.sha256!!, @@ -514,12 +518,19 @@ class OciOperationServiceImpl( } else { val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) - mediaType = if (manifest.mediaType.isNullOrEmpty()) { - HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE - } else { - manifest.mediaType!! - } - digestList = OciUtils.manifestIteratorDigest(manifest) + val mediaType = + manifest.mediaType?.ifEmpty { null } ?: ociArtifactInfo.mediaType ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE + val digestList = OciUtils.manifestIteratorDigest(manifest) + // 更新manifest节点元数据 + updateNodeMetaData( + projectId = ociArtifactInfo.projectId, + repoName = ociArtifactInfo.repoName, + version = ociArtifactInfo.reference, + fullPath = nodeDetail.fullPath, + mediaType = mediaType, + digestList = digestList, + sourceType = sourceType + ) if (ociArtifactInfo.packageName.isEmpty()) return // 处理manifest中的blob数据 syncBlobInfo( @@ -529,16 +540,6 @@ class OciOperationServiceImpl( sourceType = sourceType ) } - // 更新manifest节点元数据 - updateNodeMetaData( - projectId = ociArtifactInfo.projectId, - repoName = ociArtifactInfo.repoName, - version = ociArtifactInfo.reference, - fullPath = nodeDetail.fullPath, - mediaType = mediaType, - digestList = digestList, - sourceType = sourceType - ) } @@ -672,8 +673,8 @@ class OciOperationServiceImpl( ) // 如果当前镜像下的blob没有全部存储在制品库，则不生成版本，由定时任务去生成 if (existFlag) { - val mediaType = manifest.mediaType ?: HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) - ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE + val mediaType = + manifest.mediaType?.ifEmpty { null } ?: ociArtifactInfo.mediaType ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE val metadata = mutableMapOf( OLD_DOCKER_VERSION to ociArtifactInfo.reference, SHA256 to nodeDetail.sha256!!,