From a1e0e00f1be105e5cb1cf5a7a62e0c080245e742 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Thu, 31 Aug 2023 11:48:22 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=95=9C=E5=83=8F=E5=88=B6=E5=93=81?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E4=BC=98=E5=8C=96=20#203=20(#582)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bug: 添加第三方镜像仓库package拉取扩展点 #462 * bug: 镜像仓库支持remote代理 #462 * bug: 接口调整 #462 * bug: 引用调整 #462 * bug: 代理url校验 #462 * bug: 代码优化 #462 * bug: 引用调整 #462 * bug: 代码调整 #462 * bug: 代码调整 #462 * bug: token缓存key调整 #462 * bug: 代码调整 #462 * bug: 代码调整 #462 * feat: 代码调整 #203 * feat: blob存储节点路径调整 #203 * feat: blob刷新接口 #203 * feat: 包与版本删除逻辑调整 #203 * feat: 方法名调整 #203 * feat: 版本删除路径调整 #203 * feat: 接口调整 #203 * feat: 删除逻辑调整 #203 * feat: blob节点刷新代码调整 #203 * feat: 增加打印 #203 * feat: 获取节点路径调整 #203 * feat: 节点刷新接口逻辑调整 #203 * feat: 代码调整 #203 * feat: 代码调整 #203 * feat: 去掉注释 #203 * feat: 代码调整 #203 * feat: 禁止v1的镜像manifest文件上传,部分客户端拉取会报错 #908 * bug: feign请求字段返回先转换成String再进行类型转换 #892 (cherry picked from commit ec4614784cb5bb0b8b2149dfec094cd53df93c37) * bug: catalog相关接口调整 #203 * bug: 路径拼接调整 #203 * bug: 当包下没有版本时需要删掉包目录,清除存储在blobs目录下数据 #203 * feat: 历史数据修正方案调整 #203 * bug: 解决代码冲突#203 * bug: 代码扫描问题修复#203 * bug: 代码问题修复#203 * bug: 代码检查问题修复#203 * bug: mount上传增加元数据,用于区别新老 blob 存储路径#203 * bug: 镜像刷新前先判断是否已经是新版本路径#203 * feat: 镜像blob路径调整后,分发对应获取路径调整 #203 --- .../common/api/constant/AuthenticationKeys.kt | 34 + .../authentication}/AuthenticationProperty.kt | 2 +- .../common/api/util/AuthenticationUtil.kt | 74 ++ .../repository/remote/RemoteRepository.kt | 1 + .../common/artifact/util/http/UrlFormatter.kt | 89 ++ .../artifact/util/http/UrlFormatterTest.kt | 251 ++++++ .../bkrepo/job/batch/OciBlobNodeRefreshJob.kt | 139 +++ .../tencent/bkrepo/job/batch/RepoInitJob.kt | 26 +- .../bkrepo/job/batch/RepoRefreshJob.kt | 26 +- .../bkrepo/job/batch/base/DefaultRepoJob.kt | 1 + .../OciBlobNodeRefreshJobProperties.kt | 40 + .../config/properties/RepoJobProperties.kt | 2 +- .../com/tencent/bkrepo/oci/api/OciClient.kt | 28 +- .../bkrepo/oci/constant/OciConstants.kt | 14 +- .../bkrepo/oci/constant/OciMessageCode.kt | 5 +- .../ImagePackageInfoPullExtension.kt | 43 + .../oci/extension/ImagePackagePullContext.kt | 38 + .../oci/pojo/artifact/OciArtifactInfo.kt | 8 +- .../pojo/metadata/HelmMaintainerMetadata.kt | 43 - .../oci/pojo/remote/RemoteRequestProperty.kt | 39 + .../oci/pojo/response/ResponseProperty.kt | 40 + .../bkrepo/oci/util/OciLocationUtils.kt | 38 +- .../com/tencent/bkrepo/oci/util/OciUtils.kt | 14 - .../repository/OciRegistryLocalRepository.kt | 118 ++- .../repository/OciRegistryRemoteRepository.kt | 334 ++++--- .../resolver/OciTagArtifactInfoResolver.kt | 13 +- .../service/OciPackageController.kt | 36 +- .../oci/controller/user/CatalogController.kt | 22 +- .../oci/controller/user/OciBlobController.kt | 2 +- .../oci/controller/user/OciTagController.kt | 10 +- .../oci/controller/user/UserOciController.kt | 2 +- .../bkrepo/oci/listener/base/EventExecutor.kt | 17 +- .../consumer/RemoteImageRepoEventConsumer.kt} | 69 +- .../consumer/ReplicationEventListener.kt | 11 +- .../ThirdPartyReplicationEventConsumer.kt | 13 +- .../bkrepo/oci/service/OciOperationService.kt | 74 +- .../oci/service/impl/OciBlobServiceImpl.kt | 70 +- .../service/impl/OciOperationServiceImpl.kt | 819 +++++++++--------- .../bkrepo/oci/util/ObjectBuildUtils.kt | 13 +- .../bkrepo/oci/util/OciResponseUtils.kt | 93 +- .../resources/i18n/messages_en.properties | 5 +- .../resources/i18n/messages_zh_CN.properties | 5 +- .../resources/i18n/messages_zh_TW.properties | 5 +- .../bkrepo/replication/constant/Constants.kt | 5 +- .../handler/ArtifactReplicationHandler.kt | 6 +- ...RemoteClusterArtifactReplicationHandler.kt | 4 +- .../internal/type/DockerPackageNodeMapper.kt | 40 +- .../type/helm/HelmArtifactPushClient.kt | 4 +- .../type/oci/OciAuthorizationService.kt | 53 +- .../service/impl/RemoteNodeServiceImpl.kt | 2 +- .../bkrepo/replication/util/HttpUtils.kt | 64 +- .../bkrepo/repository/api/NodeClient.kt | 12 +- .../controller/service/NodeController.kt | 6 + .../service/node/NodeRestoreOperation.kt | 4 + .../service/node/impl/NodeRestoreSupport.kt | 7 + .../service/node/impl/NodeServiceImpl.kt | 6 + .../node/impl/edge/EdgeNodeServiceImpl.kt | 4 + .../bkrepo/repository/util/NodeQueryHelper.kt | 11 + 58 files changed, 1923 insertions(+), 1031 deletions(-) create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt rename src/backend/{replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote => common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication}/AuthenticationProperty.kt (96%) create mode 100644 src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt create mode 100644 src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt delete mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt create mode 100644 src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt rename src/backend/oci/{api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt => biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt} (53%) diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt new file mode 100644 index 0000000000..070361adef --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/constant/AuthenticationKeys.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.constant + +object AuthenticationKeys { + const val BEARER_REALM = "Bearer realm" + const val SERVICE = "service" + const val SCOPE = "scope" +} \ No newline at end of file diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt similarity index 96% rename from src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt rename to src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt index 47e4268a7f..5d10b119fe 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/pojo/remote/AuthenticationProperty.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/pojo/authentication/AuthenticationProperty.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.replication.pojo.remote +package com.tencent.bkrepo.common.api.pojo.authentication /** * 返回头中的WWW_AUTHENTICATE字段包含的属性 diff --git a/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt new file mode 100644 index 0000000000..d97bdb2e44 --- /dev/null +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/util/AuthenticationUtil.kt @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.api.util + +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.BEARER_REALM +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SCOPE +import com.tencent.bkrepo.common.api.constant.AuthenticationKeys.SERVICE +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.pojo.authentication.AuthenticationProperty + +object AuthenticationUtil { + + /** + * 解析返回头中的WWW_AUTHENTICATE字段, 只针对为Bearer realm + */ + fun parseWWWAuthenticateHeader(wwwAuthenticate: String, scope: String?): AuthenticationProperty? { + val map: MutableMap = mutableMapOf() + return try { + val params = wwwAuthenticate.split("\",") + params.forEach { + val param = it.split(Regex("="),2) + val name = param.first() + val value = param.last().replace("\"", "") + map[name] = value + } + AuthenticationProperty( + authUrl = map[BEARER_REALM]!!, + service = map[SERVICE]!!, + scope = scope + ) + } catch (e: Exception) { + null + } + } + + fun buildAuthenticationUrl(property: AuthenticationProperty, userName: String?): String? { + if (property.authUrl.isBlank()) return null + var result = if (property.authUrl.contains(StringPool.QUESTION)) { + "${property.authUrl}&$SERVICE=${property.service}" + } else { + "${property.authUrl}?$SERVICE=${property.service}" + } + property.scope?.let { + result += "&$SCOPE=${property.scope}" + } + userName?.let { result += "&account=$userName" } + return result + } +} \ No newline at end of file diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt index 5baf2589b5..7f9e3ffd5f 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt @@ -233,6 +233,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() { if (addInterceptor) { createAuthenticateInterceptor(configuration.credentials)?.let { builder.addInterceptor(it) } } + builder.retryOnConnectionFailure(true) return builder.build() } diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt index da5a5b6217..6b540cda8c 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatter.kt @@ -31,10 +31,16 @@ package com.tencent.bkrepo.common.artifact.util.http +import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.CharPool.QUESTION import com.tencent.bkrepo.common.api.constant.CharPool.SLASH +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.constant.StringPool.HTTP import com.tencent.bkrepo.common.api.constant.StringPool.HTTPS +import com.tencent.bkrepo.common.storage.innercos.retry +import java.net.HttpURLConnection +import java.net.MalformedURLException +import java.net.URL /** * Http URL 格式化工具类 @@ -82,4 +88,87 @@ object UrlFormatter { } return url } + + + /** + * 拼接url + */ + fun buildUrl( + url: String, + path: String = StringPool.EMPTY, + params: String = StringPool.EMPTY, + ): String { + if (url.isBlank()) + throw IllegalArgumentException("Url should not be blank") + val newUrl = addProtocol(url.trim().trimEnd(SLASH)) + val baseUrl = URL(newUrl, newUrl.path) + val builder = StringBuilder(baseUrl.toString().trimEnd(SLASH)) + if (path.isNotBlank()) { + builder.append(SLASH).append(path.trimStart(SLASH)) + } + if (!newUrl.query.isNullOrEmpty()) { + builder.append(QUESTION).append(newUrl.query) + } + return addParams(builder.toString(), params) + } + + fun addParams(url: String, params: String): String { + val baseUrl = URL(url) + val builder = StringBuilder(baseUrl.toString()) + + if (params.isNotEmpty()) { + if (builder.contains(QUESTION)) { + builder.append(CharPool.AND).append(params) + } else { + builder.append(QUESTION).append(params) + } + } + return builder.toString() + } + + /** + * 当没有protocol时进行添加 + */ + fun addProtocol(registry: String): URL { + try { + return URL(registry) + } catch (ignore: MalformedURLException) { + } + return addProtocolToHost(registry) + } + + /** + * 针对url如果没传protocol, 则默认以https请求发送; + * 如果http请求无法访问,则以http发送 + */ + private fun addProtocolToHost(registry: String): URL { + val url = try { + URL("$HTTPS$registry") + } catch (ignore: MalformedURLException) { + throw IllegalArgumentException("Check your input url!") + } + return try { + retry(times = 3, delayInSeconds = 1) { + validateHttpsProtocol(url) + url + } + } catch (ignore: Exception) { + URL(url.toString().replaceFirst("^https".toRegex(), "http")) + } + } + + /** + * 验证registry是否支持https + */ + private fun validateHttpsProtocol(url: URL): Boolean { + return try { + val http: HttpURLConnection = url.openConnection() as HttpURLConnection + http.instanceFollowRedirects = false + http.responseCode + http.disconnect() + true + } catch (e: Exception) { + throw e + } + } } diff --git a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt new file mode 100644 index 0000000000..1ddb218236 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/util/http/UrlFormatterTest.kt @@ -0,0 +1,251 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.util.http + +import com.tencent.bkrepo.common.api.constant.StringPool +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertAll + +internal class UrlFormatterTest { + + @Test + fun buildUrl() { + val url = "" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + try { + val result = UrlFormatter.buildUrl(url, path, params) + } catch (e: Exception) { + assertTrue(e.message!!.contains("Url should not be blank")) + } + + } + + @Test + fun buildUrl1() { + val url = "bkrepo.example.com" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl2() { + val url = "http://bkrepo.example.com" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl3() { + val url = "http://bkrepo.example.com/" + val path = StringPool.EMPTY + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com", result)} + ) + } + + @Test + fun buildUrl4() { + val url = "http://bkrepo.example.com//" + val path = "/v2/" + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2/", result)} + ) + } + + @Test + fun buildUrl5() { + val url = "http://bkrepo.example.com//" + val path = "v2" + val params = StringPool.EMPTY + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2", result)} + ) + } + + @Test + fun buildUrl6() { + val url = "http://bkrepo.example.com//" + val path = "v2" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2?a=a", result)} + ) + } + + @Test + fun buildUrl7() { + val url = "http://bkrepo.example.com/?b=b" + val path = "v2" + val params = "a=a" + + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/v2?b=b&a=a", result)} + ) + } + + @Test + fun buildUrl8() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2" + + val result = UrlFormatter.buildUrl(url, path) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2", result)} + ) + } + + @Test + fun buildUrl9() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2/" + + val result = UrlFormatter.buildUrl(url, path) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/", result)} + ) + } + + @Test + fun buildUrl10() { + val url = "http://bkrepo.example.com/test/" + val path = "/v2/" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/?a=a", result)} + ) + } + + @Test + fun buildUrl11() { + val url = "http://bkrepo.example.com/test/?b=b" + val path = "/v2/" + val params = "a=a" + val result = UrlFormatter.buildUrl(url, path, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/v2/?b=b&a=a", result)} + ) + } + + @Test + fun addParams() { + val url = "http://bkrepo.example.com/test/?b=b" + val params = "a=a" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b&a=a", result)} + ) + } + + @Test + fun addParams1() { + val url = "http://bkrepo.example.com/test/?b=b" + val params = "" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b", result)} + ) + } + + @Test + fun addParams2() { + val url = "http://bkrepo.example.com/test/" + val params = "b=b" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/?b=b", result)} + ) + } + + @Test + fun addParams3() { + val url = "http://bkrepo.example.com/test" + val params = "b=b" + val result = UrlFormatter.addParams(url, params) + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test?b=b", result)} + ) + } + + + + @Test + fun addProtocol() { + val url = "bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol1() { + val url = "http://bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("http://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol2() { + val url = "https://bkrepo.example.com/test/" + val result = UrlFormatter.addProtocol(url).toString() + assertAll( + {Assertions.assertEquals("https://bkrepo.example.com/test/", result)} + ) + } + + @Test + fun addProtocol3() { + val url = "bkrepo.example.com:test/" + try { + val result = UrlFormatter.addProtocol(url).toString() + } catch (e: Exception) { + assertTrue(e.message!!.contains("Check your input url!")) + } + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt new file mode 100644 index 0000000000..68a139804a --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt @@ -0,0 +1,139 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch + +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.job.TYPE +import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.config.properties.OciBlobNodeRefreshJobProperties +import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.find +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component + +/** + * 用于将存储在blobs目录下的公共blob节点全部迁移到对应版本目录下, + * 即从/{packageName}/blobs/xxx 到/{packageName}/blobs/{tag}/xxx + */ +@Component +@EnableConfigurationProperties(OciBlobNodeRefreshJobProperties::class) +class OciBlobNodeRefreshJob( + private val properties: OciBlobNodeRefreshJobProperties, + private val mongoTemplate: MongoTemplate, + private val ociClient: OciClient +) : DefaultContextMongoDbJob(properties) { + private val types: List + get() = properties.repositoryTypes + + override fun start(): Boolean { + return super.start() + } + + override fun entityClass(): Class { + return PackageData::class.java + } + + override fun collectionNames(): List { + return listOf(COLLECTION_NAME) + } + + override fun buildQuery(): Query { + return Query( + Criteria.where(TYPE).`in`(properties.repositoryTypes) + ) + } + + override fun run(row: PackageData, collectionName: String, context: JobContext) { + with(row) { + try { + val result = mongoTemplate.find>( + Query(Criteria(PACKAGE_ID).isEqualTo(row.id)), + PACKAGE_VERSION_NAME + ) + if (result.isEmpty()) return + var refreshStatus = true + for (map in result) { + val version = map[NAME] as String? ?: continue + logger.info( + "Preparing to send blob refresh request for package ${row.name}|${version}" + + " in repo ${row.projectId}|${row.repoName}." + ) + refreshStatus = refreshStatus && ociClient.blobPathRefresh( + projectId = row.projectId, + repoName = row.repoName, + packageName = row.name, + version = version + ).data ?: false + } + if (refreshStatus) { + // 当包下版本对应镜像的 blob节点都刷新完成后,删除旧路径下的 blob文件 + logger.info( + "Will delete blobs folder of package ${row.name}" + + " in repo ${row.projectId}|${row.repoName}." + ) + ociClient.deleteBlobsFolderAfterRefreshed(projectId, repoName, row.name) + } + } catch (e: Exception) { + throw JobExecuteException( + "Failed to send blob refresh request for package ${row.name}" + + " in repo ${row.projectId}|${row.repoName}, error: ${e.message}", e + ) + } + } + } + + data class PackageData(private val map: Map) { + val id: String by map + val repoName: String by map + val projectId: String by map + val name: String by map + val key: String by map + val type: String by map + } + + companion object { + private val logger = LoggerHolder.jobLogger + const val COLLECTION_NAME = "package" + private const val PACKAGE_VERSION_NAME = "package_version" + private const val METADATA_KEY = "blobPathRefreshed" + private const val METADATA = "metadata" + private const val PACKAGE_ID = "packageId" + private const val NAME = "name" + + } + + override fun mapToEntity(row: Map): PackageData { + return PackageData(row) + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt index e4ba11d578..d796260574 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoInitJob.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.batch import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.helm.api.HelmClient @@ -38,6 +39,7 @@ import com.tencent.bkrepo.job.batch.base.DefaultRepoJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.RepoInitJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -51,7 +53,8 @@ import java.time.LocalDateTime @EnableConfigurationProperties(RepoInitJobProperties::class) class RepoInitJob( private val properties: RepoInitJobProperties, - private val helmClient: HelmClient + private val helmClient: HelmClient, + private val ociClient: OciClient ) : DefaultRepoJob(properties) { private val types: List @@ -74,16 +77,21 @@ class RepoInitJob( } override fun run(row: ProxyRepoData, collectionName: String, context: JobContext) { - with(row) { - try { - val config = configuration.readJsonString() - if (checkConfigType(config)) { - logger.info("Init request will be sent in repo $projectId|$name") - helmClient.initIndexAndPackage(projectId, name) + try { + val config = row.configuration.readJsonString() + if (!checkConfigType(config)) return + logger.info("init request will be sent in repo ${row.projectId}|${row.name}") + when (row.type) { + RepositoryType.HELM.name -> { + helmClient.initIndexAndPackage(row.projectId, row.name) } - } catch (e: Exception) { - throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) + RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { + ociClient.getPackagesFromThirdPartyRepo(row.projectId, row.name) + } + else -> throw UnsupportedOperationException() } + } catch (e: Exception) { + throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt index 23a1776c85..b786b23a8a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/RepoRefreshJob.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.batch import com.tencent.bkrepo.common.api.util.readJsonString +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.helm.api.HelmClient @@ -37,6 +38,7 @@ import com.tencent.bkrepo.job.batch.base.DefaultRepoJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.config.properties.RepoRefreshJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query @@ -49,7 +51,8 @@ import org.springframework.stereotype.Component @EnableConfigurationProperties(RepoRefreshJobProperties::class) class RepoRefreshJob( private val properties: RepoRefreshJobProperties, - private val helmClient: HelmClient + private val helmClient: HelmClient, + private val ociClient: OciClient ) : DefaultRepoJob(properties) { private val types: List @@ -70,16 +73,21 @@ class RepoRefreshJob( } override fun run(row: ProxyRepoData, collectionName: String, context: JobContext) { - with(row) { - try { - val config = configuration.readJsonString() - if (checkConfigType(config)) { - logger.info("Refresh request will be sent in repo $projectId|$name") - helmClient.refreshIndexYamlAndPackage(projectId, name) + try { + val config = row.configuration.readJsonString() + if (!checkConfigType(config)) return + logger.info("Refresh request will be sent in repo ${row.projectId}|${row.name}") + when (row.type) { + RepositoryType.HELM.name -> { + helmClient.refreshIndexYamlAndPackage(row.projectId, row.name) } - } catch (e: Exception) { - throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) + RepositoryType.OCI.name, RepositoryType.DOCKER.name -> { + ociClient.getPackagesFromThirdPartyRepo(row.projectId, row.name) + } + else -> throw UnsupportedOperationException() } + } catch (e: Exception) { + throw JobExecuteException("Failed to send refresh request for repo ${row.projectId}|${row.name}.", e) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt index 77fa5df3a4..13c006d39c 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/DefaultRepoJob.kt @@ -66,6 +66,7 @@ abstract class DefaultRepoJob( data class ProxyRepoData(private val map: Map) { val name: String by map val projectId: String by map + val type: String by map val configuration: String by map } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt new file mode 100644 index 0000000000..f22e3f3f96 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/OciBlobNodeRefreshJobProperties.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties("job.oci-blob-node-refresh") +class OciBlobNodeRefreshJobProperties( + /** + * 需要进行远端分发集群推送的仓库类型 + * */ + var repositoryTypes: List = listOf("OCI", "DOCKER"), + override var enabled: Boolean = true, + override var cron: String = "0 0 4/24 * * ?" + ) : MongodbJobProperties() diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt index 778f07edca..386f585b21 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/RepoJobProperties.kt @@ -36,5 +36,5 @@ open class RepoJobProperties( /** * 需要特殊处理的仓库类型 * */ - var repositorytypes: List = listOf("HELM") + var repositorytypes: List = listOf("HELM", "OCI", "DOCKER") ) : MongodbJobProperties() diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt index 380d3027dd..b911c0df8a 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt @@ -34,10 +34,12 @@ import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import org.springframework.cloud.openfeign.FeignClient import org.springframework.context.annotation.Primary +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RequestMapping - +import org.springframework.web.bind.annotation.RequestParam @Api("oci") @@ -51,4 +53,28 @@ interface OciClient { fun packageCreate( @RequestBody record: OciReplicationRecordInfo ): Response + + @ApiOperation("定时从第三方仓库拉取对应的package信息") + @PostMapping("/pull/package/{projectId}/{repoName}") + fun getPackagesFromThirdPartyRepo( + @PathVariable projectId: String, + @PathVariable repoName: String + ): Response + + @ApiOperation("刷新对应版本镜像的blob节点路径") + @PostMapping("/blob/path/refresh/{projectId}/{repoName}") + fun blobPathRefresh( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam packageName: String, + @RequestParam version: String, + ): Response + + @ApiOperation("当历史数据刷新完成后,删除blobs路径下的公共blob节点") + @DeleteMapping("/blobs/delete/{projectId}/{repoName}") + fun deleteBlobsFolderAfterRefreshed( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam packageName: String + ): Response } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt index 9b2a5201dc..c4e56a21fe 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciConstants.kt @@ -42,9 +42,6 @@ const val HTTP_FORWARDED_PROTO = "X-Forwarded-Proto" const val HTTP_PROTOCOL_HTTP = "http" const val HTTP_PROTOCOL_HTTPS = "https" const val HOST = "Host" -const val BEARER_REALM = "Bearer realm" -const val SERVICE = "service" -const val SCOPE = "scope" const val PATCH = "PATCH" const val POST = "POST" const val NODE_FULL_PATH = "fullPath" @@ -53,6 +50,10 @@ const val N = "n" const val DOCKER_LINK = "Link" const val PROXY_URL = "proxyUrl" +const val BLOB_PATH_VERSION_KEY = "blobPathVersion" +const val BLOB_PATH_VERSION_VALUE = "v1" + +const val BLOB_PATH_REFRESHED_KEY = "blobPathRefreshed" const val MANIFEST = "manifest.json" const val MEDIA_TYPE = "mediaType" @@ -94,6 +95,7 @@ const val LAST_MODIFIED_BY = "lastModifiedBy" const val LAST_MODIFIED_DATE = "lastModifiedDate" const val DOWNLOADS = "downloads" const val MD5 = "md5" +const val DELETED = "deleted" const val EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" @@ -101,6 +103,10 @@ const val EMPTY_FILE_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca const val OCI_MANIFEST = "manifest.json" const val STAGE_TAG = "stageTag" +const val TAG_LIST_REQUEST = "tagList" +const val CATALOG_REQUEST = "catalog" +const val REQUEST_IMAGE = "image" + // OCIScheme is the URL scheme for OCI-based requests const val OCI_SCHEME = "oci" @@ -121,6 +127,8 @@ const val LEGACY_CHART_LAYER_MEDIA_TYPE = "application/tar+gzip" const val OCI_IMAGE_MANIFEST_MEDIA_TYPE = "application/vnd.oci.image.manifest.v1+json" +const val DOCKER_DISTRIBUTION_MANIFEST_V2 = "application/vnd.docker.distribution.manifest.v2+json" + // Content Descriptor const val CONTENT_DESCRIPTOR_MEDIA_TYPE = "application/vnd.oci.descriptor.v1+json" diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt index 5805b903cf..c4b89dd37f 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/constant/OciMessageCode.kt @@ -37,7 +37,10 @@ enum class OciMessageCode(private val key: String) : MessageCode { OCI_DELETE_RULES("oci.delete.rules"), OCI_VERSION_NOT_FOUND("oci.version.not.found"), OCI_MANIFEST_INVALID("oci.manifest.invalid"), - OCI_DIGEST_INVALID("oci.digest.invalid") + OCI_DIGEST_INVALID("oci.digest.invalid"), + OCI_MANIFEST_SCHEMA1_NOT_SUPPORT("oci.manifest.schema1.not.support"), + OCI_REMOTE_CONFIGURATION_ERROR("oci.remote.configuration.error"), + OCI_REMOTE_CREDENTIALS_INVALID("oci.remote.credentials.invalid") ; override fun getBusinessCode() = ordinal + 1 override fun getKey() = key diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt new file mode 100644 index 0000000000..e5658715d9 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackageInfoPullExtension.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.extension + +import com.tencent.devops.plugin.api.ExtensionPoint + +/** + * 拉取并存储第三方仓库下的镜像对应package信息扩展点 + */ +interface ImagePackageInfoPullExtension: ExtensionPoint { + + /** + * 拉取第三方镜像仓库package信息,并存储到对应仓库中 + */ + fun queryAndCreateDockerPackageInfo( + context: ImagePackagePullContext + ) +} diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt new file mode 100644 index 0000000000..72d0ba4b88 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/extension/ImagePackagePullContext.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.extension + +import java.net.URL + +data class ImagePackagePullContext( + val projectId: String, + val repoName:String, + val remoteUrl: URL, + val userName: String? = null, + val password: String? = null +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt index 9229cc4541..ad86c29317 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/artifact/OciArtifactInfo.kt @@ -60,11 +60,14 @@ open class OciArtifactInfo( const val BOLBS_UPLOAD_SECOND_STEP_URL = "/v2/{projectId}/{repoName}/**/blobs/uploads/{uuid}" // tags get - const val TAGS_URL = "/v2/{projectId}/{repoName}/**/tags/list" + const val TAGS_LIST_SUFFIX = "/tags/list" + // Retrieve a sorted, json list of repositories available in the registry. + const val DOCKER_CATALOG_SUFFIX = "/v2/_catalog" // version详情获取 const val OCI_VERSION_DETAIL = "/version/detail/{projectId}/{repoName}" + // 额外的package或者version 删除接口 const val OCI_PACKAGE_DELETE_URL = "/package/delete/{projectId}/{repoName}" const val OCI_VERSION_DELETE_URL = "/version/delete/{projectId}/{repoName}" @@ -72,6 +75,7 @@ open class OciArtifactInfo( const val OCI_USER_LAYER_SUFFIX = "/layer/{projectId}/{repoName}/**/{id}" const val OCI_USER_REPO_SUFFIX = "/repo/{projectId}/{repoName}" const val OCI_USER_TAG_SUFFIX = "/tag/{projectId}/{repoName}/**" - const val DOCKER_CATALOG_SUFFIX = "_catalog" + + const val OCI_BLOB_NODE_FULLPATH_REFRESH = "/blob/node/refresh" } } diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt deleted file mode 100644 index 762591e162..0000000000 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmMaintainerMetadata.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.tencent.bkrepo.oci.pojo.metadata - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonInclude - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -data class HelmMaintainerMetadata( - val name: String?, - val email: String?, - val url: String? -) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt new file mode 100644 index 0000000000..c6c9363060 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/remote/RemoteRequestProperty.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.pojo.remote + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.oci.constant.REQUEST_IMAGE + +data class RemoteRequestProperty( + var url: String, + var imageName: String, + var fullPath: String = StringPool.EMPTY, + var params: String = StringPool.EMPTY, + var type: String = REQUEST_IMAGE +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt new file mode 100644 index 0000000000..5715b5c5b3 --- /dev/null +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/response/ResponseProperty.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.oci.pojo.response + +import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.oci.pojo.digest.OciDigest + +data class ResponseProperty( + val digest: OciDigest? = null, + val location: String? = null, + val uuid: String? = null, + val range: Long? = null, + val status: HttpStatus? = null, + val contentLength: Int? = null +) diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt index 550222ab67..c7b1dbb7c2 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciLocationUtils.kt @@ -27,45 +27,47 @@ package com.tencent.bkrepo.oci.util +import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.oci.constant.OCI_MANIFEST import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest -import org.slf4j.LoggerFactory object OciLocationUtils { - private val logger = LoggerFactory.getLogger(OciLocationUtils::class.java) - fun buildManifestPath(packageName: String, tag: String): String { - return "/$packageName/manifest/$tag/$OCI_MANIFEST" + return buildManifestVersionFolderPath(packageName, tag) + OCI_MANIFEST } - fun buildDigestManifestPathWithReference(packageName: String, reference: String): String { - return buildDigestManifestPath(packageName, OciDigest(reference)) + fun buildManifestVersionFolderPath(packageName: String, tag: String): String { + return buildManifestFolderPath(packageName) +"$tag/" } - fun buildDigestManifestPathWithSha256(packageName: String, sha256: String): String { - return buildDigestManifestPath(packageName, OciDigest.fromSha256(sha256)) + fun buildManifestFolderPath(packageName: String): String { + return "/$packageName/manifest/" } - private fun buildDigestManifestPath(packageName: String, ref: OciDigest): String { - return buildPath(packageName, ref, "manifest") + fun buildDigestManifestPathWithReference(packageName: String, reference: String): String { + return buildDigestManifestPath(packageName, OciDigest(reference)) } - fun buildDigestBlobsPath(packageName: String, digestStr: String): String { - return buildPath(packageName, OciDigest(digestStr), "blobs") + private fun buildDigestManifestPath(packageName: String, ref: OciDigest): String { + return buildPath(packageName, ref, "manifest") } fun buildDigestBlobsPath(packageName: String, ref: OciDigest): String { return buildPath(packageName, ref, "blobs") } + fun buildBlobsFolderPath(packageName: String): String { + return buildPath(packageName, null, "blobs") + } + fun buildDigestBlobsUploadPath(packageName: String, ref: OciDigest): String { return buildPath(packageName, ref, "_uploads") } - private fun buildPath(packageName: String, ref: OciDigest, type: String): String { - return "/$packageName/$type/" + ref.fileName() + private fun buildPath(packageName: String, ref: OciDigest? = null, type: String): String { + return "/$packageName/$type/"+ (ref?.fileName() ?: StringPool.EMPTY) } fun manifestLocation(digest: OciDigest, ociArtifactInfo: OciArtifactInfo): String { @@ -86,6 +88,14 @@ object OciLocationUtils { } } + fun blobVersionPathLocation(reference: String, packageName: String, fileName: String): String { + return blobVersionFolderLocation(reference, packageName)+ fileName + } + + fun blobVersionFolderLocation(reference: String, packageName: String): String { + return "/$packageName/blobs/$reference/" + } + private fun returnPathLocation(digest: OciDigest, ociArtifactInfo: OciArtifactInfo, type: String): String { with(ociArtifactInfo) { return "/$packageName/$type/$digest" diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt index 402247b48f..29004f80ff 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciUtils.kt @@ -30,20 +30,15 @@ package com.tencent.bkrepo.oci.util import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.api.util.readJsonString -import com.tencent.bkrepo.common.api.util.readYamlString -import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 -import com.tencent.bkrepo.oci.constant.FILE_EXTENSION import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.model.Descriptor import com.tencent.bkrepo.oci.model.ManifestSchema1 import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.SchemaVersion -import com.tencent.bkrepo.oci.pojo.metadata.HelmChartMetadata -import com.tencent.bkrepo.oci.util.DecompressUtil.getArchivesContent import org.apache.logging.log4j.util.Strings import java.io.InputStream @@ -93,15 +88,6 @@ object OciUtils { } } - fun parseChartInputStream(inputStream: InputStream): HelmChartMetadata { - val result = inputStream.getArchivesContent(FILE_EXTENSION) - return result.byteInputStream().readYamlString() - } - - fun convertToMap(chartInfo: HelmChartMetadata): Map { - return chartInfo.toJsonString().readJsonString() - } - fun manifestIterator(manifest: ManifestSchema2): List { val list = mutableListOf() list.add(manifest.config) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt index 98722292ea..320e05b323 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryLocalRepository.kt @@ -68,6 +68,7 @@ import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest import com.tencent.bkrepo.oci.pojo.response.CatalogResponse +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.OciLocationUtils @@ -128,29 +129,13 @@ class OciRegistryLocalRepository( */ override fun onUpload(context: ArtifactUploadContext) { logger.info("Preparing to upload the oci file in repo ${context.artifactInfo.getRepoIdentify()}") - val requestMethod = context.request.method - if (PATCH == requestMethod) { - patchUpload(context) - } else { - val (digest, location) = if (POST == requestMethod) { - postUpload(context) - } else { - putUpload(context) - } - logger.info( - "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded " + - "and will can be accessed in $location" + - " in repo ${context.artifactInfo.getRepoIdentify()}" - ) - if (digest == null || location.isNullOrEmpty()) return - val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) - OciResponseUtils.buildUploadResponse( - domain = domain, - digest = digest, - locationStr = location, - response = context.response - ) - } + val responseProperty = when (context.request.method) { + PATCH -> patchUpload(context) + POST -> postUpload(context) + else -> putUpload(context) + } ?: return + val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) + OciResponseUtils.buildUploadResponse(domain, responseProperty, context.response) } /** @@ -161,27 +146,24 @@ class OciRegistryLocalRepository( * 2:Upload the chunks (PATCH) * 3:Close the session (PUT) */ - private fun patchUpload(context: ArtifactUploadContext) { + private fun patchUpload(context: ArtifactUploadContext): ResponseProperty? { logger.info("Will using patch ways to upload file in repo ${context.artifactInfo.getRepoIdentify()}") - if (context.artifactInfo !is OciBlobArtifactInfo) return + if (context.artifactInfo !is OciBlobArtifactInfo) return null with(context.artifactInfo as OciBlobArtifactInfo) { val range = context.request.getHeader("Content-Range") val length = context.request.contentLength - val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) if (!range.isNullOrEmpty() && length > -1) { logger.info("range $range, length $length, uuid $uuid") val (start, end) = getRangeInfo(range) // 判断要上传的长度是否超长 if (end - start > length - 1) { - OciResponseUtils.buildBlobUploadPatchResponse( - domain = domain, - uuid = uuid!!, - locationStr = OciLocationUtils.blobUUIDLocation(uuid!!, this), - response = HttpContextHolder.getResponse(), + return ResponseProperty( + location = OciLocationUtils.blobUUIDLocation(uuid!!, this), + status = HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE, range = length.toLong(), - status = HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE + uuid = uuid!!, + contentLength = 0 ) - return } } val patchLen = storageService.append( @@ -189,12 +171,12 @@ class OciRegistryLocalRepository( artifactFile = context.getArtifactFile(), storageCredentials = context.repositoryDetail.storageCredentials ) - OciResponseUtils.buildBlobUploadPatchResponse( - domain = domain, + return ResponseProperty( + location = OciLocationUtils.blobUUIDLocation(uuid!!, this), + status = HttpStatus.ACCEPTED, + range = patchLen, uuid = uuid!!, - locationStr = OciLocationUtils.blobUUIDLocation(uuid!!, this), - response = HttpContextHolder.getResponse(), - range = patchLen + contentLength = 0 ) } } @@ -203,7 +185,7 @@ class OciRegistryLocalRepository( * blob 上传,直接使用post * Pushing a blob monolithically :A single POST request */ - private fun postUpload(context: ArtifactUploadContext): Pair { + private fun postUpload(context: ArtifactUploadContext): ResponseProperty? { val artifactFile = context.getArtifactFile() val digest = OciDigest.fromSha256(artifactFile.getFileSha256()) ociOperationService.storeArtifact( @@ -211,13 +193,19 @@ class OciRegistryLocalRepository( artifactFile = artifactFile, storageCredentials = context.storageCredentials ) + val blobLocation = OciLocationUtils.blobLocation(digest, context.artifactInfo as OciArtifactInfo) logger.info( "Artifact ${context.artifactInfo.getArtifactFullPath()} has " + - "been uploaded to ${context.artifactInfo.getArtifactFullPath()}" + + "been uploaded to ${context.artifactInfo.getArtifactFullPath()}, " + + "and will can be accessed in $blobLocation" + " in repo ${context.artifactInfo.getRepoIdentify()}" ) - val blobLocation = OciLocationUtils.blobLocation(digest, context.artifactInfo as OciArtifactInfo) - return Pair(digest, blobLocation) + return ResponseProperty( + digest = digest, + location = blobLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** @@ -226,14 +214,12 @@ class OciRegistryLocalRepository( * 2 blob POST PATCH with PUT 上传的put模块处理 * 3 manifest PUT上传的逻辑处理 */ - private fun putUpload(context: ArtifactUploadContext): Pair { - if (context.artifactInfo is OciBlobArtifactInfo) { - return putUploadBlob(context) + private fun putUpload(context: ArtifactUploadContext): ResponseProperty? { + return when (context.artifactInfo) { + is OciBlobArtifactInfo -> putUploadBlob(context) + is OciManifestArtifactInfo -> putUploadManifest(context) + else -> null } - if (context.artifactInfo is OciManifestArtifactInfo) { - return putUploadManifest(context) - } - return Pair(null, null) } /** @@ -241,7 +227,7 @@ class OciRegistryLocalRepository( * 1 blob POST with PUT 上传的put模块处理 * 2 blob POST PATCH with PUT 上传的put模块处理 */ - private fun putUploadBlob(context: ArtifactUploadContext): Pair { + private fun putUploadBlob(context: ArtifactUploadContext): ResponseProperty { val artifactInfo = context.artifactInfo as OciBlobArtifactInfo val sha256 = artifactInfo.getDigestHex() val fileInfo = try { @@ -279,19 +265,25 @@ class OciRegistryLocalRepository( } } val digest = OciDigest.fromSha256(fileInfo.sha256) + val blobLocation = OciLocationUtils.blobLocation(digest, artifactInfo) logger.info( "Artifact ${context.artifactInfo.getArtifactFullPath()} " + "has been uploaded to ${context.artifactInfo.getArtifactFullPath()}" + + "and will can be accessed in $blobLocation" + " in repo ${context.artifactInfo.getRepoIdentify()}" ) - val blobLocation = OciLocationUtils.blobLocation(digest, artifactInfo) - return Pair(digest, blobLocation) + return ResponseProperty( + digest = digest, + location = blobLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** * manifest文件 PUT上传的逻辑处理 */ - private fun putUploadManifest(context: ArtifactUploadContext): Pair { + private fun putUploadManifest(context: ArtifactUploadContext): ResponseProperty { val artifactInfo = context.artifactInfo as OciManifestArtifactInfo val artifactFile = context.getArtifactFile() val digest = OciDigest.fromSha256(artifactFile.getFileSha256()) @@ -300,19 +292,25 @@ class OciRegistryLocalRepository( artifactFile = artifactFile, storageCredentials = context.storageCredentials ) - logger.info( - "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded to ${node!!.fullPath}" + - " in repo ${context.artifactInfo.getRepoIdentify()}" - ) - // 上传manifest文件,同时需要将manifest中对应blob的属性进行补充到blob节点中,同时创建package相关信息 + // 上传manifest文件,同时需要判断manifest中的blob节点是否已经存在,同时创建package相关信息 ociOperationService.updateOciInfo( ociArtifactInfo = artifactInfo, digest = digest, storageCredentials = context.storageCredentials, - nodeDetail = node + nodeDetail = node!! ) val manifestLocation = OciLocationUtils.manifestLocation(digest, artifactInfo) - return Pair(digest, manifestLocation) + logger.info( + "Artifact ${context.artifactInfo.getArtifactFullPath()} has been uploaded to ${node.fullPath}" + + "and will can be accessed in $manifestLocation" + + " in repo ${context.artifactInfo.getRepoIdentify()}" + ) + return ResponseProperty( + digest = digest, + location = manifestLocation, + status = HttpStatus.CREATED, + contentLength = 0 + ) } /** diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index e44181617b..8a6833e65b 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -31,14 +31,21 @@ package com.tencent.bkrepo.oci.artifact.repository +import com.google.common.cache.CacheBuilder import com.tencent.bkrepo.common.api.constant.BEARER_AUTH_PREFIX +import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpHeaders.WWW_AUTHENTICATE import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.api.util.AuthenticationUtil +import com.tencent.bkrepo.common.api.util.BasicAuthUtils import com.tencent.bkrepo.common.api.util.JsonUtils +import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext @@ -50,31 +57,35 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.artifactStream import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter -import com.tencent.bkrepo.oci.constant.BEARER_REALM +import com.tencent.bkrepo.oci.constant.CATALOG_REQUEST +import com.tencent.bkrepo.oci.constant.DOCKER_DISTRIBUTION_MANIFEST_V2 import com.tencent.bkrepo.oci.constant.DOCKER_LINK import com.tencent.bkrepo.oci.constant.LAST_TAG import com.tencent.bkrepo.oci.constant.MEDIA_TYPE import com.tencent.bkrepo.oci.constant.N import com.tencent.bkrepo.oci.constant.OCI_API_PREFIX +import com.tencent.bkrepo.oci.constant.OCI_FILTER_ENDPOINT import com.tencent.bkrepo.oci.constant.OCI_IMAGE_MANIFEST_MEDIA_TYPE import com.tencent.bkrepo.oci.constant.OciMessageCode -import com.tencent.bkrepo.oci.constant.SCOPE -import com.tencent.bkrepo.oci.constant.SERVICE +import com.tencent.bkrepo.oci.constant.PROXY_URL +import com.tencent.bkrepo.oci.constant.TAG_LIST_REQUEST import com.tencent.bkrepo.oci.exception.OciForbiddenRequestException import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciBlobArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.auth.BearerToken import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.remote.RemoteRequestProperty import com.tencent.bkrepo.oci.pojo.response.CatalogResponse +import com.tencent.bkrepo.oci.pojo.response.OciResponse import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import okhttp3.Credentials import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response @@ -83,6 +94,7 @@ import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.io.InputStream import java.net.URL +import java.util.concurrent.TimeUnit import java.util.regex.Pattern import javax.ws.rs.core.UriBuilder @@ -91,9 +103,15 @@ class OciRegistryRemoteRepository( private val ociOperationService: OciOperationService ) : RemoteRepository() { + private val clientCache = CacheBuilder.newBuilder().maximumSize(100) + .expireAfterWrite(60, TimeUnit.MINUTES).build() + private val tokenCache = CacheBuilder.newBuilder().maximumSize(100) + .expireAfterWrite(60, TimeUnit.MINUTES).build() + + override fun upload(context: ArtifactUploadContext) { with(context) { - val message = "Forbidden to upload chart into a remote repository [$projectId/$repoName]" + val message = "Forbidden to upload artifact into a remote repository [$projectId/$repoName]" logger.warn(message) throw OciForbiddenRequestException(OciMessageCode.OCI_FILE_UPLOAD_FORBIDDEN, "$projectId|$repoName") } @@ -103,8 +121,13 @@ class OciRegistryRemoteRepository( * 下载制品 */ override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { - return getCacheArtifactResource(context) ?: run { - return doRequest(context) as ArtifactResource? + return if (context.artifactInfo is OciManifestArtifactInfo) { + // 同一镜像tag可能被覆盖更新,对于manifest.json文件每次都去远端拉取 + doRequest(context) as ArtifactResource? + } else { + getCacheArtifactResource(context) ?: run { + doRequest(context) as ArtifactResource? + } } } @@ -122,35 +145,68 @@ class OciRegistryRemoteRepository( */ private fun doRequest(context: ArtifactContext): Any? { val remoteConfiguration = context.getRemoteConfiguration() - val httpClient = createHttpClient(remoteConfiguration, false) - val downloadUrl = createRemoteDownloadUrl(context) + val httpClient = clientCache.getIfPresent(remoteConfiguration) ?: run { + clientCache.put(remoteConfiguration, createHttpClient(remoteConfiguration, false)) + clientCache.getIfPresent(remoteConfiguration) + } + val property = getRemoteRequestProperty(context) + val downloadUrl = createRemoteDownloadUrl(context, property) logger.info("Remote request $downloadUrl will be sent") - val request = buildRequest(downloadUrl, remoteConfiguration) - val response = httpClient.newCall(request).execute() - var responseWithAuth: Response? = null + val tokenKey = buildTokenCacheKey( + context.getStringAttribute(PROXY_URL)!!,remoteConfiguration.credentials.username, property.imageName + ) + val request = buildRequest(downloadUrl, remoteConfiguration, tokenCache.getIfPresent(tokenKey)) try { - if (response.isSuccessful) return onResponse(context, response) - // 针对返回401进行token获取 - val token = getAuthenticationCode(response, remoteConfiguration, httpClient) - if (token.isNullOrBlank()) return null - val requestWithAuth = buildRequest( - url = downloadUrl, - configuration = remoteConfiguration, - addBasicInterceptor = false, - token = token - ) - responseWithAuth = httpClient - .newCall(requestWithAuth) - .execute() + httpClient!!.newCall(request).execute().use { + if (it.isSuccessful) return onResponse(context, it) + if (it.code != HttpStatus.UNAUTHORIZED.value) { + logger.warn("response code is ${it.code} for url $downloadUrl") + return null + } + return doRequestWithToken( + context = context, + wwwAuthenticate = it.header(WWW_AUTHENTICATE), + remoteConfiguration = remoteConfiguration, + imageName = property.imageName, + tokenKey = tokenKey, + downloadUrl = downloadUrl + ) + } + } catch (e: Exception) { + logger.error("Error occurred while sending request $downloadUrl", e) + throw NodeNotFoundException(downloadUrl) + } + } + + /** + * 当返回401时,按照docker标准协议去拉取token,然后进行文件下载 + */ + private fun doRequestWithToken( + context: ArtifactContext, + wwwAuthenticate: String?, + remoteConfiguration: RemoteConfiguration, + imageName: String, + tokenKey: String, + downloadUrl: String, + ): Any? { + // 针对返回401进行token获取 + val proxyUrl = context.getStringAttribute(PROXY_URL)!! + val token = getAuthenticationCode(proxyUrl, wwwAuthenticate, remoteConfiguration, imageName) ?: return null + tokenCache.put(tokenKey, token) + val requestWithToken = buildRequest( + url = downloadUrl, + configuration = remoteConfiguration, + addBasicInterceptor = false, + token = token + ) + clientCache.getIfPresent(remoteConfiguration)!!.newCall(requestWithToken).execute().use {responseWithAuth -> return if (checkResponse(responseWithAuth)) { onResponse(context, responseWithAuth) } else null - } finally { - response.body?.close() - responseWithAuth?.body?.close() } } + private fun onResponse(context: ArtifactContext, response: Response): Any? { if (context is ArtifactDownloadContext) { return onDownloadResponse(context, response) @@ -171,11 +227,15 @@ class OciRegistryRemoteRepository( addBasicInterceptor: Boolean = true ): Request { val requestBuilder = Request.Builder().url(url) - if (addBasicInterceptor) { + if (addBasicInterceptor && token.isNullOrEmpty()) { requestBuilder.addInterceptor(configuration) } else { token?.let { requestBuilder.header(HttpHeaders.AUTHORIZATION, token) } } + // 拉取第三方仓库时,默认会返回v1版本的镜像格式 + if (url.contains("/manifests/")) { + requestBuilder.header(HttpHeaders.ACCEPT, DOCKER_DISTRIBUTION_MANIFEST_V2) + } return requestBuilder.build() } @@ -183,7 +243,7 @@ class OciRegistryRemoteRepository( val username = configuration.credentials.username val password = configuration.credentials.password if (username != null && password != null) { - val credentials = Credentials.basic(username, password) + val credentials = BasicAuthUtils.encode(username, password) this.header(HttpHeaders.AUTHORIZATION, credentials) } return this @@ -193,110 +253,158 @@ class OciRegistryRemoteRepository( * 生成远程构件下载url */ override fun createRemoteDownloadUrl(context: ArtifactContext): String { - val configuration = context.getRemoteConfiguration() - if (context.artifactInfo is OciBlobArtifactInfo) { - val artifactInfo = context.artifactInfo as OciBlobArtifactInfo - return createUrl( - url = configuration.url, - fullPath = OciLocationUtils.blobPathLocation(artifactInfo.getDigest(), artifactInfo), - params = StringPool.EMPTY - ) - } - if (context.artifactInfo is OciManifestArtifactInfo) { - val artifactInfo = context.artifactInfo as OciManifestArtifactInfo - return createUrl( - url = configuration.url, - fullPath = OciLocationUtils.manifestPathLocation(artifactInfo.reference, artifactInfo), - params = StringPool.EMPTY - ) + val property = getRemoteRequestProperty(context) + return createRemoteDownloadUrl(context, property) + } + + fun createRemoteDownloadUrl(context: ArtifactContext, property: RemoteRequestProperty): String { + return when (property.type) { + CATALOG_REQUEST -> createCatalogUrl(property) + TAG_LIST_REQUEST -> createTagListUrl(property) + else -> createUrl(property) } - if (context.artifactInfo is OciTagArtifactInfo) { - val artifactInfo = context.artifactInfo as OciTagArtifactInfo - if (artifactInfo.packageName.isBlank()) { - val (_, params) = createParamsForTagList(context) - return createCatalogUrl(configuration.url, params) - } else { - val (fullPath, params) = createParamsForTagList(context) - return createUrl(configuration.url, fullPath, params) + } + + /** + * 获取不同情况下对应的请求属性 + */ + private fun getRemoteRequestProperty(context: ArtifactContext): RemoteRequestProperty { + val configuration = context.getRemoteConfiguration() + val url = UrlFormatter.addProtocol(configuration.url).toString() + context.putAttribute(PROXY_URL, url) + return when (context.artifactInfo) { + is OciBlobArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciBlobArtifactInfo + RemoteRequestProperty( + url = url, + fullPath = OciLocationUtils.blobPathLocation(artifactInfo.getDigest(), artifactInfo), + imageName = artifactInfo.packageName + ) } + is OciTagArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciTagArtifactInfo + if (artifactInfo.packageName.isBlank()) { + val (_, params) = createParamsForTagList(context) + RemoteRequestProperty( + url = url, + params = params, + type = CATALOG_REQUEST, + imageName = StringPool.EMPTY + ) + } else { + val (fullPath, params) = createParamsForTagList(context) + RemoteRequestProperty( + url = url, + fullPath = fullPath, + params = params, + type = TAG_LIST_REQUEST, + imageName = artifactInfo.packageName + ) + } + } + is OciManifestArtifactInfo -> { + val artifactInfo = context.artifactInfo as OciManifestArtifactInfo + RemoteRequestProperty( + url = url, + fullPath = OciLocationUtils.manifestPathLocation(artifactInfo.reference, artifactInfo), + imageName = artifactInfo.packageName + ) + } + else -> RemoteRequestProperty(url = url, imageName = StringPool.EMPTY) } - return createUrl(configuration.url) } /** * 拼接url */ - private fun createUrl(url: String, fullPath: String = StringPool.EMPTY, params: String = StringPool.EMPTY): String { - val baseUrl = URL(url) - val v2Url = URL(baseUrl, "/v2" + baseUrl.path) - return UrlFormatter.format(v2Url.toString(), fullPath, params) + private fun createUrl(property: RemoteRequestProperty): String { + with(property) { + val baseUrl = URL(url) + val v2Url = URL(baseUrl, OCI_FILTER_ENDPOINT + baseUrl.path) + return UrlFormatter.format(v2Url.toString(), fullPath, params) + } } /** * 拼接catalog url */ - private fun createCatalogUrl(url: String, params: String = StringPool.EMPTY): String { - val baseUrl = URL(url) - val builder = UriBuilder.fromPath(OCI_API_PREFIX) - .host(baseUrl.host).scheme(baseUrl.protocol) - .path(DOCKER_CATALOG_SUFFIX) - .queryParam(params) - return builder.build().toString() + private fun createCatalogUrl(property: RemoteRequestProperty): String { + with(property) { + return UrlFormatter.buildUrl(url, DOCKER_CATALOG_SUFFIX, params) + } + } + + /** + * 拼接tag list url + */ + private fun createTagListUrl(property: RemoteRequestProperty): String { + with(property) { + val url = UriBuilder.fromUri(url) + .path(OCI_API_PREFIX) + .path(imageName) + .path(TAGS_LIST_SUFFIX) + .build().toString() + return UrlFormatter.addParams(url, params) + } } private fun getAuthenticationCode( - response: Response, + proxyUrl: String, + wwwAuthenticate: String?, configuration: RemoteConfiguration, - httpClient: OkHttpClient + imageName: String ): String? { - if (response.code != HttpStatus.UNAUTHORIZED.value) { + if (wwwAuthenticate.isNullOrBlank() || !wwwAuthenticate.startsWith(BEARER_AUTH_PREFIX)) { + logger.warn("response wwwAuthenticate header $wwwAuthenticate is illegal") return null } - val wwwAuthenticate = response.header(WWW_AUTHENTICATE) - if (wwwAuthenticate.isNullOrBlank() || !wwwAuthenticate.startsWith(BEARER_AUTH_PREFIX)) { + val scope = getScope(proxyUrl, imageName) + val authProperty = AuthenticationUtil.parseWWWAuthenticateHeader(wwwAuthenticate, scope) + if (authProperty == null) { + logger.warn("Auth url can not be parsed from header $wwwAuthenticate!") return null } - val url = parseWWWAuthenticateHeader(wwwAuthenticate) - logger.info("The url for authenticating is $url") - if (url.isNullOrEmpty()) return null + val urlStr = AuthenticationUtil.buildAuthenticationUrl(authProperty, configuration.credentials.username) + logger.info("The url for authentication is $urlStr") + if (urlStr.isNullOrEmpty()) return null val request = buildRequest( - url = url, + url = urlStr, configuration = configuration, - addBasicInterceptor = false + addBasicInterceptor = true ) - val tokenResponse = httpClient.newCall(request).execute() - try { - if (!tokenResponse.isSuccessful) return null - val body = tokenResponse.body!! - val artifactFile = createTempFile(body) - val size = artifactFile.getSize() - val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size)) - artifactFile.delete() - val bearerToken = JsonUtils.objectMapper.readValue(artifactStream, BearerToken::class.java) - return "Bearer ${bearerToken.token}" - } finally { - tokenResponse.body?.close() + clientCache.getIfPresent(configuration)!!.newCall(request).execute().use { + if (!it.isSuccessful) { + val error = try { + JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java).toJsonString() + } catch (ignore: Exception) { + StringPool.EMPTY + } + val errMsg = "Could not get token from auth service," + + " code is ${it.code} and response is $error" + logger.warn(errMsg) + throw ErrorCodeException( + OciMessageCode.OCI_REMOTE_CREDENTIALS_INVALID, + errMsg + ) + } + try { + val bearerToken = JsonUtils.objectMapper.readValue(it.body!!.byteStream(), BearerToken::class.java) + return "Bearer ${bearerToken.token}" + } catch (e: Exception) { + throw ErrorCodeException( + OciMessageCode.OCI_REMOTE_CONFIGURATION_ERROR, + "Could not get token from auth service, please check your remote configuration!" + ) + } } } - /** - * 解析返回头中的WWW_AUTHENTICATE字段, 只针对为Bearer realm - */ - private fun parseWWWAuthenticateHeader(wwwAuthenticate: String): String? { - val map: MutableMap = mutableMapOf() - return try { - val params = wwwAuthenticate.split(",") - params.forEach { - val param = it.split("=") - val name = param.first() - val value = param.last().removeSurrounding("\"") - map[name] = value - } - "${map[BEARER_REALM]}?$SERVICE=${map[SERVICE]}&$SCOPE=${map[SCOPE]}" - } catch (e: Exception) { - logger.warn("Parsing wwwAuthenticate header error: ${e.message}") - null - } + + private fun getScope(remoteUrl: String, imageName: String): String { + val baseUrl = URL(remoteUrl) + val target = baseUrl.path.removePrefix(StringPool.SLASH) + .removeSuffix(StringPool.SLASH) + StringPool.SLASH + imageName + return "repository:$target:pull" } private fun createParamsForTagList(context: ArtifactContext): Pair { @@ -331,7 +439,7 @@ class OciRegistryRemoteRepository( inputStream = artifactStream, artifactName = context.artifactInfo.getResponseName(), node = node, - channel = ArtifactChannel.LOCAL + channel = ArtifactChannel.PROXY ) return buildResponse( cacheNode = node, @@ -412,13 +520,14 @@ class OciRegistryRemoteRepository( // 针对manifest文件获取会通过tag或manifest获取,避免重复创建 fullPath?.let { val node = nodeClient.getNodeDetail(ociArtifactInfo.projectId, ociArtifactInfo.repoName, fullPath).data - if (node != null) return node + if (node != null && artifactFile.getFileSha256() == node.sha256) return node } + val url = context.getStringAttribute(PROXY_URL) var nodeDetail = ociOperationService.storeArtifact( ociArtifactInfo = ociArtifactInfo, artifactFile = artifactFile, storageCredentials = context.storageCredentials, - proxyUrl = configuration.url + proxyUrl = url ) // 针对manifest文件需要更新metadata if (context.artifactInfo is OciManifestArtifactInfo) { @@ -435,7 +544,7 @@ class OciRegistryRemoteRepository( private fun updateManifestAndBlob(context: ArtifactDownloadContext, nodeDetail: NodeDetail) { with(context.artifactInfo as OciManifestArtifactInfo) { val digest = OciDigest.fromSha256(nodeDetail.sha256!!) - // 上传manifest文件,同时需要将manifest中对应blob的属性进行补充到blob节点中,同时创建package相关信息 + // 上传manifest文件,同时创建package相关信息 ociOperationService.updateOciInfo( ociArtifactInfo = this, digest = digest, @@ -506,6 +615,11 @@ class OciRegistryRemoteRepository( return n } + private fun buildTokenCacheKey(remoteUrl: String, userName: String?, imageName: String): String { + val scope = getScope(remoteUrl, imageName) + return "$remoteUrl${CharPool.COLON}$scope${CharPool.COLON}$userName" + } + companion object { val logger: Logger = LoggerFactory.getLogger(OciRegistryRemoteRepository::class.java) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt index 0dbd8a7f01..d5830211a3 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/resolver/OciTagArtifactInfoResolver.kt @@ -38,13 +38,12 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.artifact.resolve.path.ArtifactInfoResolver import com.tencent.bkrepo.common.artifact.resolve.path.Resolver import com.tencent.bkrepo.oci.constant.OCI_TAG -import com.tencent.bkrepo.oci.constant.USER_API_PREFIX import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo -import io.undertow.servlet.spec.HttpServletRequestImpl -import javax.servlet.http.HttpServletRequest import org.springframework.stereotype.Component import org.springframework.web.servlet.HandlerMapping +import javax.servlet.http.HttpServletRequest @Component @Resolver(OciTagArtifactInfo::class) @@ -58,17 +57,14 @@ class OciTagArtifactInfoResolver : ArtifactInfoResolver { ): ArtifactInfo { val requestURL = ArtifactContextHolder.getUrlPath(this.javaClass.name)!! return when { - requestURL.contains(TAG_PREFIX) -> { + requestURL.contains(TAGS_LIST_SUFFIX) -> { val requestUrl = request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE).toString() - val packageName = requestUrl.removePrefix("$USER_API_PREFIX/tag/$projectId/$repoName/") + val packageName = requestUrl.removePrefix("/$projectId/$repoName/v2/").removeSuffix(TAGS_LIST_SUFFIX) validate(packageName) val tag = request.getParameter(OCI_TAG) ?: StringPool.EMPTY OciTagArtifactInfo(projectId, repoName, packageName, tag) } requestURL.contains(DOCKER_CATALOG_SUFFIX) -> { - val params = (request as HttpServletRequestImpl).queryParameters - val projectId = params?.get("projectId")?.first ?: StringPool.EMPTY - val repoName = params?.get("repoName")?.first ?: StringPool.EMPTY OciTagArtifactInfo(projectId, repoName, StringPool.EMPTY, StringPool.EMPTY) } else -> { @@ -89,6 +85,5 @@ class OciTagArtifactInfoResolver : ArtifactInfoResolver { companion object { const val PACKAGE_NAME_PATTERN = "[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*" - const val TAG_PREFIX = "/ext/tag/" } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt index aa7bbe8695..8aaaccd54c 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt @@ -28,9 +28,12 @@ package com.tencent.bkrepo.oci.controller.service import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.artifact.event.repo.RepoCreatedEvent +import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.oci.api.OciClient import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao +import com.tencent.bkrepo.oci.listener.base.EventExecutor import com.tencent.bkrepo.oci.model.TOciReplicationRecord import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.third.OciReplicationRecordInfo @@ -42,8 +45,9 @@ import org.springframework.web.bind.annotation.RestController @RestController class OciPackageController( private val operationService: OciOperationService, - private val ociReplicationRecordDao: OciReplicationRecordDao -): OciClient { + private val ociReplicationRecordDao: OciReplicationRecordDao, + private val eventExecutor: EventExecutor + ): OciClient { override fun packageCreate(record: OciReplicationRecordInfo): Response { with(record) { val ociArtifactInfo = OciManifestArtifactInfo( @@ -65,4 +69,32 @@ class OciPackageController( return ResponseBuilder.success() } } + + override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String): Response { + eventExecutor.submit(RepoCreatedEvent( + projectId = projectId, + repoName = repoName, + userId = SecurityUtils.getUserId() + )) + return ResponseBuilder.success() + } + + override fun blobPathRefresh( + projectId: String, repoName: String, packageName: String, version: String + ): Response { + return ResponseBuilder.success( + operationService.refreshBlobNode( + projectId = projectId, + repoName = repoName, + pName = packageName, + pVersion = version + )) + } + + override fun deleteBlobsFolderAfterRefreshed( + projectId: String, repoName: String, packageName: String + ): Response { + operationService.deleteBlobsFolderAfterRefreshed(projectId, repoName, packageName) + return ResponseBuilder.success() + } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt index b7b8a0f4cc..d70aa77742 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/CatalogController.kt @@ -34,11 +34,11 @@ package com.tencent.bkrepo.oci.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.security.permission.Permission -import com.tencent.bkrepo.oci.config.OciProperties import com.tencent.bkrepo.oci.constant.DOCKER_API_VERSION import com.tencent.bkrepo.oci.constant.DOCKER_HEADER_API_VERSION import com.tencent.bkrepo.oci.constant.DOCKER_LINK -import com.tencent.bkrepo.oci.constant.OCI_FILTER_ENDPOINT +import com.tencent.bkrepo.oci.constant.OCI_PROJECT_ID +import com.tencent.bkrepo.oci.constant.OCI_REPO_NAME import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.DOCKER_CATALOG_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.service.OciCatalogService @@ -47,29 +47,25 @@ import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController @RestController -@RequestMapping(OCI_FILTER_ENDPOINT) -class CatalogController( - private val catalogService: OciCatalogService, - private val ociProperties: OciProperties -) { +class CatalogController(private val catalogService: OciCatalogService) { /** * 返回仓库下所有image名列表 */ - @GetMapping(DOCKER_CATALOG_SUFFIX) + @GetMapping("/{projectId}/{repoName}$DOCKER_CATALOG_SUFFIX") @Permission(type = ResourceType.REPO, action = PermissionAction.WRITE) fun list( artifactInfo: OciTagArtifactInfo, - @RequestParam(required = true) - @ApiParam(required = true) + @PathVariable + @ApiParam(value = OCI_PROJECT_ID, required = true) projectId: String, - @RequestParam(required = true) - @ApiParam(required = true) + @PathVariable + @ApiParam(value = OCI_REPO_NAME, required = true) repoName: String, @RequestParam(required = false) @ApiParam(value = "n", required = false) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt index 9a5e57ce73..0f0f3a8ac9 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciBlobController.kt @@ -106,7 +106,7 @@ class OciBlobController( } /** - * 删除manifest文件 + * 删除blob文件 * 只能通过digest删除 */ @DeleteMapping(BOLBS_URL) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt index e4c265832b..c005855fdc 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/OciTagController.kt @@ -30,9 +30,8 @@ package com.tencent.bkrepo.oci.controller.user import com.tencent.bkrepo.auth.pojo.enums.PermissionAction import com.tencent.bkrepo.auth.pojo.enums.ResourceType import com.tencent.bkrepo.common.security.permission.Permission -import com.tencent.bkrepo.oci.config.OciProperties import com.tencent.bkrepo.oci.constant.DOCKER_LINK -import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_URL +import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo.Companion.TAGS_LIST_SUFFIX import com.tencent.bkrepo.oci.pojo.artifact.OciTagArtifactInfo import com.tencent.bkrepo.oci.pojo.tags.TagsInfo import com.tencent.bkrepo.oci.service.OciTagService @@ -48,15 +47,12 @@ import org.springframework.web.bind.annotation.RestController * oci tag controller */ @RestController -class OciTagController( - private val ociTagService: OciTagService, - private val ociProperties: OciProperties -) { +class OciTagController(private val ociTagService: OciTagService) { /** * 获取blob对应的tag信息 */ @Permission(type = ResourceType.REPO, action = PermissionAction.READ) - @RequestMapping(TAGS_URL, method = [RequestMethod.GET]) + @RequestMapping("/{projectId}/{repoName}/**$TAGS_LIST_SUFFIX", method = [RequestMethod.GET]) fun getTagList( artifactInfo: OciTagArtifactInfo, @RequestParam n: Int?, diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt index 8394e2d629..3d24080a5f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/user/UserOciController.kt @@ -56,7 +56,6 @@ import com.tencent.bkrepo.oci.service.OciOperationService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam -import javax.servlet.http.HttpServletRequest import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable @@ -65,6 +64,7 @@ import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController import org.springframework.web.servlet.HandlerMapping +import javax.servlet.http.HttpServletRequest @Suppress("MVCPathVariableInspection") @Api("oci产品接口") diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt index e743922882..c8bcdf504e 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/base/EventExecutor.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.oci.listener.base import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel @@ -39,10 +40,12 @@ import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.Logger import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component import java.util.concurrent.Future import java.util.concurrent.ThreadPoolExecutor -open class EventExecutor( +@Component +class EventExecutor( open val nodeClient: NodeClient, open val repositoryClient: RepositoryClient, open val ociOperationService: OciOperationService @@ -57,7 +60,7 @@ open class EventExecutor( ): Future { return threadPoolExecutor.submit { try { - replicationEventHandler(event) + eventHandler(event) true } catch (exception: Throwable) { logger.warn("Error occurred while executing the oci event: $exception") @@ -66,6 +69,16 @@ open class EventExecutor( } } + private fun eventHandler(event: ArtifactEvent) { + when (event.type) { + EventType.REPLICATION_THIRD_PARTY -> replicationEventHandler(event) + EventType.REPO_CREATED, EventType.REPO_REFRESHED, EventType.REPO_UPDATED -> { + ociOperationService.getPackagesFromThirdPartyRepo(event.projectId, event.repoName) + } + else -> throw UnsupportedOperationException() + } + } + private fun replicationEventHandler(event: ArtifactEvent) { with(event) { val packageName = event.data["packageName"].toString() diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt similarity index 53% rename from src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt rename to src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt index 011fa99f2b..753b5fd69b 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/pojo/metadata/HelmChartMetadata.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/RemoteImageRepoEventConsumer.kt @@ -25,44 +25,43 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.oci.pojo.metadata +package com.tencent.bkrepo.oci.listener.consumer -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonInclude -import com.github.zafarkhaja.semver.Version +import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType +import com.tencent.bkrepo.oci.listener.base.EventExecutor +import org.slf4j.LoggerFactory +import org.springframework.messaging.Message +import org.springframework.stereotype.Component +import java.util.function.Consumer -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -data class HelmChartMetadata( - var apiVersion: String?, - var appVersion: String?, - var created: String?, - var deprecated: Boolean?, - var description: String?, - var digest: String?, - var engine: String?, - var home: String?, - var icon: String?, - var keywords: List = emptyList(), - var maintainers: List = emptyList(), - var name: String, - var sources: List = emptyList(), - var urls: List = emptyList(), - var version: String, - var type: String?, - var annotations: Map? -) : Comparable { +/** + * 构件事件消费者,用于实时同步 + * 对应destination为对应ArtifactEvent.topic + */ +@Component("remoteOciRepo") +class RemoteImageRepoEventConsumer( + private val eventExecutor: EventExecutor +) : Consumer>{ + + /** + * 允许接收的事件类型 + */ + private val acceptTypes = setOf( + EventType.REPO_CREATED, + EventType.REPO_UPDATED, + EventType.REPO_REFRESHED + ) - override fun compareTo(other: HelmChartMetadata): Int { - val result = this.name.compareTo(other.name) - return if (result != 0) { - result - } else { - try { - Version.valueOf(other.version).compareWithBuildsTo(Version.valueOf(this.version)) - } catch (ignored: Exception) { - other.version.compareTo(this.version) - } + override fun accept(message: Message) { + if (!acceptTypes.contains(message.payload.type)) { + return } + logger.info("current repo operation message header is ${message.headers}") + eventExecutor.submit(message.payload) + } + + companion object { + private val logger = LoggerFactory.getLogger(RemoteImageRepoEventConsumer::class.java) } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt index 5ae0f39e02..7a7e928732 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ReplicationEventListener.kt @@ -29,9 +29,6 @@ package com.tencent.bkrepo.oci.listener.consumer import com.tencent.bkrepo.common.artifact.event.replication.ThirdPartyReplicationEvent import com.tencent.bkrepo.oci.listener.base.EventExecutor -import com.tencent.bkrepo.oci.service.OciOperationService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.springframework.context.event.EventListener import org.springframework.stereotype.Component @@ -40,15 +37,13 @@ import org.springframework.stereotype.Component */ @Component class ReplicationEventListener( - override val nodeClient: NodeClient, - override val repositoryClient: RepositoryClient, - override val ociOperationService: OciOperationService -): EventExecutor(nodeClient, repositoryClient, ociOperationService) { + private val eventExecutor: EventExecutor +){ /** * 第三方同步事件处理 */ @EventListener(ThirdPartyReplicationEvent::class) fun handle(event: ThirdPartyReplicationEvent) { - submit(event) + eventExecutor.submit(event) } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt index 9dad95d058..bad477b59f 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/listener/consumer/ThirdPartyReplicationEventConsumer.kt @@ -30,9 +30,6 @@ package com.tencent.bkrepo.oci.listener.consumer import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent import com.tencent.bkrepo.common.artifact.event.base.EventType import com.tencent.bkrepo.oci.listener.base.EventExecutor -import com.tencent.bkrepo.oci.service.OciOperationService -import com.tencent.bkrepo.repository.api.NodeClient -import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.messaging.Message import org.springframework.stereotype.Component @@ -45,12 +42,8 @@ import java.util.function.Consumer */ @Component("thirdPartyReplication") class ThirdPartyReplicationEventConsumer( - override val nodeClient: NodeClient, - override val repositoryClient: RepositoryClient, - override val ociOperationService: OciOperationService -) : - Consumer>, - EventExecutor(nodeClient, repositoryClient, ociOperationService) { + private val eventExecutor: EventExecutor +) : Consumer> { /** * 允许接收的事件类型 @@ -64,7 +57,7 @@ class ThirdPartyReplicationEventConsumer( return } logger.info("current third party replication message header is ${message.headers}") - submit(message.payload) + eventExecutor.submit(message.payload) } companion object { diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt index ad696b23e6..dd63c325d3 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/OciOperationService.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.oci.service import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel +import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.pojo.artifact.OciArtifactInfo @@ -39,44 +40,10 @@ import com.tencent.bkrepo.oci.pojo.response.OciImageResult import com.tencent.bkrepo.oci.pojo.response.OciTagResult import com.tencent.bkrepo.oci.pojo.user.PackageVersionInfo import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import javax.servlet.http.HttpServletRequest interface OciOperationService { - /** - * 保存节点元数据 - */ - fun saveMetaData( - projectId: String, - repoName: String, - fullPath: String, - metadata: MutableMap, - userId: String - ) - - /** - * 当mediatype为CHART_LAYER_MEDIA_TYPE,需要解析chart.yaml文件 - */ - fun loadArtifactInput( - chartDigest: String?, - projectId: String, - repoName: String, - packageName: String, - version: String, - storageCredentials: StorageCredentials? - ): Map? - - /** - * 需要将blob中相关metadata写进package version中 - */ - fun updatePackageInfo( - ociArtifactInfo: OciArtifactInfo, - packageKey: String, - appVersion: String? = null, - description: String? = null - ) - /** * 查询包版本详情 */ @@ -121,11 +88,6 @@ interface OciOperationService { manifestPath: String, ): Boolean - /** - * 当使用追加上传时,文件已存储,只需存储节点信息 - */ - fun createNode(request: NodeCreateRequest, storageCredentials: StorageCredentials?): NodeDetail - /** * 保存文件内容(当使用追加上传时,文件已存储,只需存储节点信息) * 特殊:对于manifest文件,node存tag @@ -145,13 +107,14 @@ interface OciOperationService { fun getNodeFullPath(artifactInfo: OciArtifactInfo): String? /** - * 根据sha256值获取对应的node fullpath,md5,size + * 根据sha256值和path获取对应的node fullpath,md5,size */ fun getNodeByDigest( projectId: String, repoName: String, - digestStr: String - ): NodeProperty + digestStr: String, + path: String? = null + ): NodeProperty? /** * 针对老的docker仓库的数据做兼容性处理 @@ -193,4 +156,31 @@ interface OciOperationService { pageSize: Int, tag: String? ): OciTagResult + + /** + * 拉取第三方镜像仓库package信息 + */ + fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String) + + /** + * oci blob 路径调整,由/packageName/blobs/XXX -> /packageName/blobs/version/XXX + */ + fun refreshBlobNode( + projectId: String, + repoName: String, + pName: String, + pVersion: String, + userId: String = SecurityUtils.getUserId() + ): Boolean + + + /** + * 当package下所有版本的blob路径都刷新到新的路径下后,删除/packageName/blobs目录 + */ + fun deleteBlobsFolderAfterRefreshed( + projectId: String, + repoName: String, + pName: String, + userId: String = SecurityUtils.getUserId() + ) } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt index 07d76fa503..53646481db 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciBlobServiceImpl.kt @@ -43,17 +43,22 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadConte import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.service.util.HttpContextHolder import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE import com.tencent.bkrepo.oci.constant.OciMessageCode import com.tencent.bkrepo.oci.constant.REPO_TYPE import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.pojo.artifact.OciBlobArtifactInfo import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import com.tencent.bkrepo.oci.service.OciBlobService import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.ObjectBuildUtils import com.tencent.bkrepo.oci.util.OciLocationUtils import com.tencent.bkrepo.oci.util.OciResponseUtils +import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @@ -61,6 +66,7 @@ import org.springframework.stereotype.Service class OciBlobServiceImpl( private val storage: StorageService, private val repoClient: RepositoryClient, + private val nodeClient: NodeClient, private val ociOperationService: OciOperationService, private val permissionManager: PermissionManager ) : OciBlobService { @@ -95,10 +101,15 @@ class OciBlobServiceImpl( logger.info("Will obtain uuid for uploading blobs in repo ${artifactInfo.getRepoIdentify()}.") val uuidCreated = startAppend(this) val domain = ociOperationService.getReturnDomain(HttpContextHolder.getRequest()) - OciResponseUtils.buildBlobUploadUUIDResponse( + val responseProperty = ResponseProperty( + uuid = uuidCreated, + location = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + status = HttpStatus.ACCEPTED, + contentLength = 0 + ) + OciResponseUtils.buildUploadResponse( domain, - uuidCreated, - OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + responseProperty, HttpContextHolder.getResponse() ) } else { @@ -120,50 +131,59 @@ class OciBlobServiceImpl( repoName = mountRepoName ) } catch (e: ErrorCodeException) { - val uuidCreated = startAppend(this) - OciResponseUtils.buildBlobMountResponse( - domain = domain, - locationStr = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), - status = HttpStatus.ACCEPTED, - response = HttpContextHolder.getResponse() - ) + buildSessionIdLocationForUpload(this, domain) return } } val nodeProperty = ociOperationService.getNodeByDigest( mountProjectId, mountRepoName, ociDigest.toString() - ) - if (nodeProperty.fullPath == null) { + ) ?: run { logger.warn("Could not find $ociDigest in repo $mountProjectId|$mountRepoName to mount") - val uuidCreated = startAppend(this) - OciResponseUtils.buildBlobMountResponse( - domain = domain, - locationStr = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), - status = HttpStatus.ACCEPTED, - response = HttpContextHolder.getResponse() - ) + buildSessionIdLocationForUpload(this, domain) return } + // 用于新版本 blobs 路径区分 + val metadata: MutableList = mutableListOf( + MetadataModel(key = BLOB_PATH_VERSION_KEY, value = BLOB_PATH_VERSION_VALUE, system = true) + ) val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( projectId = projectId, repoName = repoName, size = nodeProperty.size!!, sha256 = ociDigest.hex, fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, ociDigest), - md5 = nodeProperty.md5!! + md5 = nodeProperty.md5!!, + metadata = metadata ) - val repoDetail = repoClient.getRepoDetail(projectId, repoName).data - ociOperationService.createNode(nodeCreateRequest, repoDetail!!.storageCredentials) + nodeClient.createNode(nodeCreateRequest) val blobLocation = OciLocationUtils.blobLocation(ociDigest, this) - OciResponseUtils.buildBlobMountResponse( + val responseProperty = ResponseProperty( + location = blobLocation, + status = HttpStatus.CREATED + ) + OciResponseUtils.buildUploadResponse( domain = domain, - locationStr = blobLocation, - status = HttpStatus.CREATED, + responseProperty = responseProperty, response = HttpContextHolder.getResponse() ) } } + private fun buildSessionIdLocationForUpload(artifactInfo: OciBlobArtifactInfo, domain: String) { + val uuidCreated = startAppend(artifactInfo) + val responseProperty = ResponseProperty( + uuid = uuidCreated, + location = OciLocationUtils.blobUUIDLocation(uuidCreated, artifactInfo), + status = HttpStatus.ACCEPTED, + contentLength = 0 + ) + OciResponseUtils.buildUploadResponse( + domain = domain, + responseProperty = responseProperty, + response = HttpContextHolder.getResponse() + ) + } + private fun splitRepoInfo(from: String?): Pair? { if (from.isNullOrEmpty()) return null val values = from.split(CharPool.SLASH) diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt index b810b5a7a5..9bb3516df1 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/service/impl/OciOperationServiceImpl.kt @@ -32,14 +32,19 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.StreamUtils.readText import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE +import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.manager.StorageManager +import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration +import com.tencent.bkrepo.common.artifact.pojo.configuration.composite.CompositeConfiguration +import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HeaderUtils @@ -47,10 +52,10 @@ import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.FileInfo import com.tencent.bkrepo.oci.config.OciProperties -import com.tencent.bkrepo.oci.constant.APP_VERSION -import com.tencent.bkrepo.oci.constant.CHART_LAYER_MEDIA_TYPE +import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_KEY +import com.tencent.bkrepo.oci.constant.BLOB_PATH_VERSION_VALUE import com.tencent.bkrepo.oci.constant.DESCRIPTION -import com.tencent.bkrepo.oci.constant.DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1 import com.tencent.bkrepo.oci.constant.DOWNLOADS import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_BY import com.tencent.bkrepo.oci.constant.LAST_MODIFIED_DATE @@ -68,6 +73,8 @@ import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao import com.tencent.bkrepo.oci.exception.OciBadRequestException import com.tencent.bkrepo.oci.exception.OciFileNotFoundException import com.tencent.bkrepo.oci.exception.OciVersionNotFoundException +import com.tencent.bkrepo.oci.extension.ImagePackageInfoPullExtension +import com.tencent.bkrepo.oci.extension.ImagePackagePullContext import com.tencent.bkrepo.oci.model.Descriptor import com.tencent.bkrepo.oci.model.ManifestSchema2 import com.tencent.bkrepo.oci.model.TOciReplicationRecord @@ -84,6 +91,7 @@ import com.tencent.bkrepo.oci.pojo.user.PackageVersionInfo import com.tencent.bkrepo.oci.service.OciOperationService import com.tencent.bkrepo.oci.util.ObjectBuildUtils import com.tencent.bkrepo.oci.util.OciLocationUtils +import com.tencent.bkrepo.oci.util.OciLocationUtils.buildBlobsFolderPath import com.tencent.bkrepo.oci.util.OciResponseUtils import com.tencent.bkrepo.oci.util.OciUtils import com.tencent.bkrepo.repository.api.MetadataClient @@ -91,17 +99,27 @@ import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.PackageClient import com.tencent.bkrepo.repository.api.PackageMetadataClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER +import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest +import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest import com.tencent.bkrepo.repository.pojo.packages.VersionListOption import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail import com.tencent.bkrepo.repository.pojo.search.NodeQueryBuilder import com.tencent.bkrepo.repository.pojo.search.PackageQueryBuilder +import com.tencent.devops.plugin.api.PluginManager +import com.tencent.devops.plugin.api.applyExtension import org.apache.commons.lang3.StringUtils import org.slf4j.Logger import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.where import org.springframework.stereotype.Service import java.nio.charset.Charset import java.time.Instant @@ -119,8 +137,10 @@ class OciOperationServiceImpl( private val storageManager: StorageManager, private val repositoryClient: RepositoryClient, private val ociProperties: OciProperties, - private val ociReplicationRecordDao: OciReplicationRecordDao -) : OciOperationService { + private val ociReplicationRecordDao: OciReplicationRecordDao, + private val storageCredentialsClient: StorageCredentialsClient, + private val pluginManager: PluginManager + ) : OciOperationService { /** * 检查package 对应的version是否存在 @@ -142,77 +162,6 @@ class OciOperationServiceImpl( ) } - /** - * 保存节点元数据 - */ - override fun saveMetaData( - projectId: String, - repoName: String, - fullPath: String, - metadata: MutableMap, - userId: String - ) { - val metadataSaveRequest = ObjectBuildUtils.buildMetadataSaveRequest( - projectId = projectId, - repoName = repoName, - fullPath = fullPath, - metadata = metadata, - userId = userId - ) - metadataClient.saveMetadata(metadataSaveRequest) - } - - /** - * 当mediaType为CHART_LAYER_MEDIA_TYPE,需要解析chart.yaml文件 - */ - override fun loadArtifactInput( - chartDigest: String?, - projectId: String, - repoName: String, - packageName: String, - version: String, - storageCredentials: StorageCredentials? - ): Map? { - if (chartDigest.isNullOrBlank()) return null - val blobDigest = OciDigest(chartDigest) - val fullPath = OciLocationUtils.buildDigestBlobsPath(packageName, blobDigest) - nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { node -> - logger.info( - "Will read chart.yaml data from $fullPath with package $packageName " + - "and version $version under repo $projectId/$repoName" - ) - storageManager.loadArtifactInputStream(node, storageCredentials)?.let { - return try { - OciUtils.convertToMap(OciUtils.parseChartInputStream(it)) - } catch (e: Exception) { - logger.warn("Convert chart.yaml error: ${e.message}") - null - } - } - } - return null - } - - /** - * 需要将blob中相关metadata写进package version中 - */ - override fun updatePackageInfo( - ociArtifactInfo: OciArtifactInfo, - packageKey: String, - appVersion: String?, - description: String? - ) { - with(ociArtifactInfo) { - val packageUpdateRequest = ObjectBuildUtils.buildPackageUpdateRequest( - artifactInfo = this, - name = packageName, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - packageClient.updatePackage(packageUpdateRequest) - } - } /** * 删除package @@ -224,10 +173,10 @@ class OciOperationServiceImpl( val packageKey = PackageKeys.ofName(repoDetail.type.name.toLowerCase(), packageName) if (version.isNotBlank()) { packageClient.findVersionByName( - projectId, - repoName, - packageKey, - version + projectId = projectId, + repoName = repoName, + packageKey = packageKey, + version = version ).data?.let { removeVersion( artifactInfo = this, @@ -236,58 +185,26 @@ class OciOperationServiceImpl( packageKey = packageKey ) } ?: throw VersionNotFoundException(version) + if (!packageClient.listAllVersion(projectId, repoName, packageKey).data.isNullOrEmpty()) { + return + } + // 当没有版本时删除删除package目录 + deleteNode(projectId, repoName, "${StringPool.SLASH}$packageName", userId) } else { - packageClient.listAllVersion( + // 删除package目录 + deleteNode(projectId, repoName, "${StringPool.SLASH}$packageName", userId) + //删除package下所有版本 + packageClient.deletePackage( projectId, repoName, packageKey - ).data.orEmpty().forEach { - removeVersion( - artifactInfo = this, - version = it.name, - userId = userId, - packageKey = packageKey - ) - } - } - updatePackageExtension(artifactInfo, packageKey) - } - } - - /** - * 节点删除后,将package extension信息更新 - */ - private fun updatePackageExtension(artifactInfo: OciArtifactInfo, packageKey: String) { - with(artifactInfo) { - val version = packageClient.findPackageByKey(projectId, repoName, packageKey).data?.latest - try { - val chartDigest = findHelmChartYamlInfo(this, version) - val chartYaml = loadArtifactInput( - chartDigest = chartDigest, - projectId = projectId, - repoName = repoName, - packageName = packageName, - version = version!!, - storageCredentials = getRepositoryInfo(this).storageCredentials ) - // 针对helm chart包,将部分信息放入到package中 - chartYaml?.let { - val (appVersion, description) = getMetaDataFromChart(chartYaml) - updatePackageInfo( - ociArtifactInfo = artifactInfo, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - } - } catch (e: Exception) { - logger.warn("can not convert meta data") } } } /** - * 删除[version] 对应的node节点也会一起删除 + * 删除[version] 默认会删除对应的节点 */ private fun removeVersion( artifactInfo: OciArtifactInfo, @@ -296,78 +213,51 @@ class OciOperationServiceImpl( packageKey: String ) { with(artifactInfo) { - val nodeDetail = getBlobNodeDetail( - projectId = projectId, - repoName = repoName, - name = artifactInfo.packageName, - version = version - ) ?: return - // 删除manifest - deleteNode( - projectId = projectId, - repoName = repoName, - packageName = packageName, - path = nodeDetail.fullPath, + // 删除对应版本下所有关联的节点,包含manifest以及blobs + deleteVersionRelatedNodes( + artifactInfo = artifactInfo, + version = version, userId = userId ) packageClient.deleteVersion(projectId, repoName, packageKey, version) } } + /** - * 针对helm特殊处理,查找chart对应的digest,用于读取对应的chart.yaml信息 + * 删除对应版本下所有关联的节点,包含manifest以及blobs */ - private fun findHelmChartYamlInfo(artifactInfo: OciArtifactInfo, version: String? = null): String? { + private fun deleteVersionRelatedNodes( + artifactInfo: OciArtifactInfo, + version: String, + userId: String, + ) { with(artifactInfo) { - if (version.isNullOrBlank()) return null - val nodeDetail = getBlobNodeDetail( - projectId = projectId, - repoName = repoName, - name = artifactInfo.packageName, - version = version - ) ?: return null - val inputStream = storageService.load( - digest = nodeDetail.sha256!!, - range = Range.full(nodeDetail.size), - storageCredentials = getRepositoryInfo(artifactInfo).storageCredentials - ) ?: return null - try { - val manifest = OciUtils.streamToManifestV2(inputStream) - return (OciUtils.manifestIterator(manifest, CHART_LAYER_MEDIA_TYPE) ?: return null).digest - } catch (e: OciBadRequestException) { - logger.warn("Manifest convert error: ${e.message}") - } - return null + val manifestFolder = OciLocationUtils.buildManifestVersionFolderPath(packageName, version) + val blobsFolder = OciLocationUtils.blobVersionFolderLocation(version, packageName) + logger.info("Will delete blobsFolder [$blobsFolder] and manifestFolder $manifestFolder " + + "in package $packageName|$version in repo [$projectId/$repoName]") + // 删除manifestFolder + deleteNode(projectId, repoName, manifestFolder, userId) + // 删除blobs + deleteNode(projectId, repoName, blobsFolder, userId) } } + private fun deleteNode( projectId: String, repoName: String, - packageName: String, - userId: String, - digestStr: String? = null, - path: String? = null + fullPath: String, + userId: String ) { - val fullPath = path ?: digestStr?.let { - val nodeDetail = getBlobNodeDetail( - projectId = projectId, - repoName = repoName, - name = packageName, - digestStr = digestStr - ) - nodeDetail?.fullPath - } - fullPath?.let { - logger.info("Will delete node [$fullPath] with package $packageName in repo [$projectId/$repoName]") - val request = NodeDeleteRequest( - projectId = projectId, - repoName = repoName, - fullPath = fullPath, - operator = userId - ) - nodeClient.deleteNode(request) - } + val request = NodeDeleteRequest( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + operator = userId + ) + nodeClient.deleteNode(request) } /** @@ -398,7 +288,7 @@ class OciOperationServiceImpl( packageKey = packageKey, version = version ) - val nodeDetail = getBlobNodeDetail( + val nodeDetail = getImageNodeDetail( projectId = projectId, repoName = repoName, name = name, @@ -416,7 +306,7 @@ class OciOperationServiceImpl( * 获取node节点 * 查不到抛出OciFileNotFoundException异常 */ - private fun getBlobNodeDetail( + private fun getImageNodeDetail( projectId: String, repoName: String, name: String, @@ -476,14 +366,16 @@ class OciOperationServiceImpl( */ private fun buildNodeCreateRequest( ociArtifactInfo: OciArtifactInfo, - artifactFile: ArtifactFile + artifactFile: ArtifactFile, + metadata: List? = null ): NodeCreateRequest { return ObjectBuildUtils.buildNodeCreateRequest( projectId = ociArtifactInfo.projectId, repoName = ociArtifactInfo.repoName, artifactFile = artifactFile, - fullPath = ociArtifactInfo.getArtifactFullPath() + fullPath = ociArtifactInfo.getArtifactFullPath(), + metadata = metadata ) } @@ -498,42 +390,27 @@ class OciOperationServiceImpl( fileInfo: FileInfo?, proxyUrl: String? ): NodeDetail? { - val request = buildNodeCreateRequest(ociArtifactInfo, artifactFile) + // 用于新版本 blobs 路径区分, blob存储路径由 /{package}/blobs/转为/{package}/blobs/{version}/ + val metadata: MutableList = mutableListOf( + MetadataModel(key = BLOB_PATH_VERSION_KEY, value = BLOB_PATH_VERSION_VALUE, system = true) + ) + proxyUrl?.let { + metadata.add(MetadataModel(key = PROXY_URL, value = proxyUrl, system = true)) + } + val request = buildNodeCreateRequest(ociArtifactInfo, artifactFile, metadata) val nodeDetail = if (fileInfo != null) { val newNodeRequest = request.copy( size = fileInfo.size, md5 = fileInfo.md5, sha256 = fileInfo.sha256 ) - createNode(newNodeRequest, storageCredentials) - null + nodeClient.createNode(newNodeRequest).data } else { storageManager.storeArtifactFile(request, artifactFile, storageCredentials) } - proxyUrl?.let { - saveMetaData( - projectId = request.projectId, - repoName = request.repoName, - fullPath = request.fullPath, - metadata = mutableMapOf(PROXY_URL to proxyUrl), - userId = SecurityUtils.getUserId() - ) - } return nodeDetail } - /** - * 当使用追加上传时,文件已存储,只需存储节点信息 - */ - override fun createNode(request: NodeCreateRequest, storageCredentials: StorageCredentials?): NodeDetail { - try { - return nodeClient.createNode(request).data!! - } catch (exception: Exception) { - // 异常往上抛 - throw exception - } - } - /** * 更新整个blob相关信息,blob相关的mediatype,version等信息需要从manifest中获取 */ @@ -548,27 +425,20 @@ class OciOperationServiceImpl( "Will start to update oci info for ${ociArtifactInfo.getArtifactFullPath()} " + "in repo ${ociArtifactInfo.getRepoIdentify()}" ) - val manifestBytes = storageService.load( - nodeDetail.sha256.orEmpty(), - Range.full(nodeDetail.size), - storageCredentials - )!!.readText() - val schemaVersion = OciUtils.schemeVersion(manifestBytes) - // 将该版本对应的blob sha256放到manifest节点的元数据中 - var digestList: List? = null - val (mediaType, manifest) = if (schemaVersion.schemaVersion == 1) { - Pair(DOCKER_IMAGE_MANIFEST_MEDIA_TYPE_V1, null) + // https://github.com/docker/docker-ce/blob/master/components/engine/distribution/push_v2.go + // docker 客户端上传manifest时先按照schema2的格式上传, + // 如失败则按照schema1格式上传,但是非docker客户端不兼容schema1版本manifest + val manifest = loadManifest(nodeDetail.sha256!!, nodeDetail.size, storageCredentials) + ?: throw OciBadRequestException(OciMessageCode.OCI_MANIFEST_SCHEMA1_NOT_SUPPORT) + // 更新manifest文件的metadata + val mediaType = if (manifest.mediaType.isNullOrEmpty()) { + HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE } else { - val manifest = OciUtils.stringToManifestV2(manifestBytes) - // 更新manifest文件的metadata - val mediaTypeV2 = if (manifest.mediaType.isNullOrEmpty()) { - HeaderUtils.getHeader(HttpHeaders.CONTENT_TYPE) ?: OCI_IMAGE_MANIFEST_MEDIA_TYPE - } else { - manifest.mediaType - } - digestList = OciUtils.manifestIteratorDigest(manifest) - Pair(mediaTypeV2, manifest) + manifest.mediaType } + val digestList = OciUtils.manifestIteratorDigest(manifest) + + // 更新manifest节点元数据 updateNodeMetaData( projectId = ociArtifactInfo.projectId, repoName = ociArtifactInfo.repoName, @@ -578,25 +448,34 @@ class OciOperationServiceImpl( digestList = digestList, sourceType = sourceType ) - // 同步blob相关metadata - if (ociArtifactInfo.packageName.isNotEmpty()) { - if (schemaVersion.schemaVersion == 1) { - syncBlobInfoV1( - ociArtifactInfo = ociArtifactInfo, - manifestDigest = digest, - manifestPath = nodeDetail.fullPath, - sourceType = sourceType - ) - } else { - syncBlobInfo( - ociArtifactInfo = ociArtifactInfo, - manifest = manifest!!, - manifestDigest = digest, - storageCredentials = storageCredentials, - manifestPath = nodeDetail.fullPath, - sourceType = sourceType - ) - } + + + if (ociArtifactInfo.packageName.isEmpty()) return + // 处理manifest中的blob数据 + syncBlobInfo( + ociArtifactInfo = ociArtifactInfo, + manifest = manifest, + nodeDetail = nodeDetail, + sourceType = sourceType + ) + } + + + private fun loadManifest( + sha256: String, + size: Long, + storageCredentials: StorageCredentials? + ): ManifestSchema2? { + return try { + val manifestBytes = storageService.load( + sha256, + Range.full(size), + storageCredentials + )!!.readText() + + OciUtils.stringToManifestV2(manifestBytes) + } catch (e: Exception) { + null } } @@ -607,34 +486,16 @@ class OciOperationServiceImpl( with(ociArtifactInfo) { val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data ?: return false val nodeDetail = nodeClient.getNodeDetail(projectId, repoName, manifestPath).data ?: return false - val manifestBytes = storageService.load( - nodeDetail.sha256!!, - Range.full(nodeDetail.size), - repositoryDetail.storageCredentials - )!!.readText() - val manifest = OciUtils.stringToManifestV2(manifestBytes) - val descriptorList = OciUtils.manifestIterator(manifest) - // 用于判断是否所有blob都以存在 - var existFlag: Boolean - var size: Long = 0 - - descriptorList.forEach { - size += it.size - existFlag = doSyncBlob(it, ociArtifactInfo, repositoryDetail.storageCredentials) - // 如果当前镜像下的blob没有全部存储在制品库,则不生成版本,由定时任务去生成 - if (!existFlag) return false - } - // 根据flag生成package信息以及package version信息 - doPackageOperations( - manifestPath = manifestPath, + val manifest = loadManifest( + nodeDetail.sha256!!, nodeDetail.size, repositoryDetail.storageCredentials + ) ?: return false + return syncBlobInfo( ociArtifactInfo = ociArtifactInfo, - manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), - size = size, - chartYaml = null, + manifest = manifest, + nodeDetail = nodeDetail, sourceType = ArtifactChannel.REPLICATION, userId = SYSTEM_USER ) - return true } } @@ -647,7 +508,6 @@ class OciOperationServiceImpl( version: String? = null, fullPath: String, mediaType: String, - chartYaml: Map? = null, digestList: List? = null, sourceType: ArtifactChannel? = null ) { @@ -655,111 +515,127 @@ class OciOperationServiceImpl( val metadata = ObjectBuildUtils.buildMetadata( mediaType = mediaType, version = version, - yamlData = chartYaml, digestList = digestList, sourceType = sourceType ) - saveMetaData( + + updateNodeMetaData( projectId = projectId, repoName = repoName, fullPath = fullPath, - metadata = metadata, - userId = SecurityUtils.getUserId() + metadata = metadata ) } - /** - * 同步fsLayers层的数据 - */ - private fun syncBlobInfoV1( - ociArtifactInfo: OciManifestArtifactInfo, - manifestDigest: OciDigest, - manifestPath: String, - sourceType: ArtifactChannel? = null + + private fun updateNodeMetaData( + projectId: String, + repoName: String, + fullPath: String, + metadata: Map ) { - logger.info( - "Will start to sync fsLayers' blob info from manifest ${ociArtifactInfo.getArtifactFullPath()} " + - "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." - ) - // 根据flag生成package信息以及packageversion信息 - doPackageOperations( - manifestPath = manifestPath, - ociArtifactInfo = ociArtifactInfo, - manifestDigest = manifestDigest, - size = 0, - sourceType = sourceType + val metadataSaveRequest = ObjectBuildUtils.buildMetadataSaveRequest( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + metadata = metadata, + userId = SecurityUtils.getUserId() ) + try{ + metadataClient.saveMetadata(metadataSaveRequest) + } catch (ignore: Exception) { + // 并发情况下会出现节点找不到问题 + } } + + /** * 同步blob层的数据和config里面的数据 */ private fun syncBlobInfo( ociArtifactInfo: OciManifestArtifactInfo, + nodeDetail: NodeDetail, + sourceType: ArtifactChannel? = null, manifest: ManifestSchema2, - manifestDigest: OciDigest, - storageCredentials: StorageCredentials?, - manifestPath: String, - sourceType: ArtifactChannel? = null - ) { + userId: String = SecurityUtils.getUserId() + ): Boolean { logger.info( "Will start to sync blobs and config info from manifest ${ociArtifactInfo.getArtifactFullPath()} " + - "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." + "to blobs in repo ${ociArtifactInfo.getRepoIdentify()}." + ) + // existFlag 判断manifest里的所有blob是否都已经创建节点 + // size 整个镜像blob汇总的大小 + val (existFlag, size) = manifestHandler( + manifest, ociArtifactInfo, userId ) - val descriptorList = OciUtils.manifestIterator(manifest) - // 用于判断是否所有blob都以存在 - var existFlag = true - var chartYaml: Map? = null - // 统计所有manifest中的文件size作为整个package version的size - var size: Long = 0 - // 同步layer以及config层blob信息 - descriptorList.forEach { - size += it.size - chartYaml = when (it.mediaType) { - CHART_LAYER_MEDIA_TYPE -> { - // 针对helm chart,需要将chart.yaml中相关信息存入对应节点中 - loadArtifactInput( - chartDigest = it.digest, - projectId = ociArtifactInfo.projectId, - repoName = ociArtifactInfo.repoName, - packageName = ociArtifactInfo.packageName, - version = ociArtifactInfo.reference, - storageCredentials = storageCredentials - ) - } - else -> null - } - existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, storageCredentials) - } // 如果当前镜像下的blob没有全部存储在制品库,则不生成版本,由定时任务去生成 if (existFlag) { + // 第三方同步的索引更新等所有文件全部上传完成后才去进行 // 根据flag生成package信息以及package version信息 doPackageOperations( - manifestPath = manifestPath, + manifestPath = nodeDetail.fullPath, ociArtifactInfo = ociArtifactInfo, - manifestDigest = manifestDigest, + manifestDigest = OciDigest.fromSha256(nodeDetail.sha256!!), size = size, - chartYaml = chartYaml, - sourceType = sourceType + sourceType = sourceType, + userId = userId ) + return true } else { - ociReplicationRecordDao.save(TOciReplicationRecord( - projectId = ociArtifactInfo.projectId, - repoName = ociArtifactInfo.repoName, - packageName = ociArtifactInfo.packageName, - packageVersion = ociArtifactInfo.reference, - manifestPath = manifestPath - )) + val query = Query.query( + where(TOciReplicationRecord::projectId).isEqualTo(ociArtifactInfo.projectId) + .and(TOciReplicationRecord::repoName).isEqualTo(ociArtifactInfo.repoName) + .and(TOciReplicationRecord::packageName).isEqualTo(ociArtifactInfo.packageName) + .and(TOciReplicationRecord::packageVersion).isEqualTo(ociArtifactInfo.reference) + ) + val update = Update().setOnInsert(TOciReplicationRecord::manifestPath.name, nodeDetail.fullPath) + ociReplicationRecordDao.upsert(query, update) + return false + } + } + + + /** + * 针对v2版本manifest文件做特殊处理 + */ + private fun manifestHandler( + manifest: ManifestSchema2, + ociArtifactInfo: OciManifestArtifactInfo, + userId: String = SecurityUtils.getUserId() + ): Pair { + // 用于判断是否所有blob都以存在 + var existFlag = true + // 统计所有mainfest中的文件size作为整个package version的size + var size: Long = 0 + val descriptorList = OciUtils.manifestIterator(manifest) + + // 当同一版本覆盖上传时,先删除当前版本对应的blobs目录,然后再创建 + val blobsFolder = OciLocationUtils.blobVersionFolderLocation( + ociArtifactInfo.reference, ociArtifactInfo.packageName + ) + deleteNode(ociArtifactInfo.projectId, ociArtifactInfo.repoName, blobsFolder, userId) + + // 同步layer以及config层blob信息 + descriptorList.forEach { + size += it.size + existFlag = existFlag && doSyncBlob(it, ociArtifactInfo, userId) + if (!existFlag) { + // 第三方同步场景下,如果当前镜像下的blob没有全部存储在制品库,则不生成版本,由定时任务去生成 + return Pair(false, 0) + } } + return Pair(existFlag, size) } + /** * 更新blobs的信息 */ private fun doSyncBlob( descriptor: Descriptor, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials? + userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { logger.info( @@ -775,7 +651,7 @@ class OciOperationServiceImpl( fullPath = fullPath, descriptor = descriptor, ociArtifactInfo = this, - storageCredentials = storageCredentials + userId = userId ) } } @@ -787,23 +663,36 @@ class OciOperationServiceImpl( fullPath: String, descriptor: Descriptor, ociArtifactInfo: OciManifestArtifactInfo, - storageCredentials: StorageCredentials? + userId: String = SecurityUtils.getUserId() ): Boolean { with(ociArtifactInfo) { - val blobExist = nodeClient.checkExist(projectId, repoName, fullPath).data!! - // blob节点不存在,但是在同一仓库下存在digest文件 - if (!blobExist) { - val (existFullPath, md5) = getNodeByDigest(projectId, repoName, descriptor.digest) - if (existFullPath == null) return false + // 并发情况下,版本目录下可能存在着非该版本的blob + // 覆盖上传时会先删除原有目录,并发情况下可能导致blobs节点不存在 + val nodeProperty = getNodeByDigest(projectId, repoName, descriptor.digest) ?: run { + nodeClient.getDeletedNodeDetailBySha256( + projectId, repoName, descriptor.sha256).data?.let { + NodeProperty(StringPool.EMPTY, it.md5, it.size) + } ?: return false + } + val newPath = OciLocationUtils.blobVersionPathLocation(reference, packageName, descriptor.filename) + if (newPath != nodeProperty.fullPath) { + // 创建新路径节点 /packageName/blobs/version/xxx val nodeCreateRequest = ObjectBuildUtils.buildNodeCreateRequest( projectId = projectId, repoName = repoName, - size = descriptor.size, + size = nodeProperty.size!!, sha256 = descriptor.sha256, - fullPath = fullPath, - md5 = md5 ?: StringPool.UNKNOWN + fullPath = newPath, + md5 = nodeProperty.md5 ?: StringPool.UNKNOWN, + userId = userId ) - createNode(nodeCreateRequest, storageCredentials) + nodeClient.createNode(nodeCreateRequest) + } + val metadataMap = metadataClient.listMetadata(projectId, repoName, fullPath).data + if (metadataMap?.get(BLOB_PATH_VERSION_KEY) != null) { + // 只有当新建的blob路径节点才去删除,历史的由定时任务去刷新然后删除 + // 删除临时存储路径节点 /packageName/blobs/xxx + deleteNode(projectId, repoName, fullPath, userId) } return true } @@ -817,7 +706,6 @@ class OciOperationServiceImpl( ociArtifactInfo: OciManifestArtifactInfo, manifestDigest: OciDigest, size: Long, - chartYaml: Map? = null, sourceType: ArtifactChannel? = null, userId: String = SecurityUtils.getUserId() ) { @@ -828,7 +716,6 @@ class OciOperationServiceImpl( val packageKey = PackageKeys.ofName(repoType.toLowerCase(), packageName) val metadata = mutableMapOf(MANIFEST_DIGEST to manifestDigest.toString()) .apply { - chartYaml?.let { this.putAll(chartYaml) } sourceType?.let { this[SOURCE_TYPE] = sourceType } } val request = ObjectBuildUtils.buildPackageVersionCreateRequest( @@ -848,17 +735,6 @@ class OciOperationServiceImpl( version = ociArtifactInfo.reference, metadata = metadata ) - - // 针对helm chart包,将部分信息放入到package中 - chartYaml?.let { - val (appVersion, description) = getMetaDataFromChart(chartYaml) - updatePackageInfo( - ociArtifactInfo = ociArtifactInfo, - appVersion = appVersion, - description = description, - packageKey = packageKey - ) - } } } @@ -880,36 +756,28 @@ class OciOperationServiceImpl( metadata = metadata, userId = SecurityUtils.getUserId() ) - packageMetadataClient.saveMetadata(metadataSaveRequest) - } - - /** - * 针对helm chart包,将部分信息放入到package中 - */ - private fun getMetaDataFromChart(chartYaml: Map? = null): Pair { - var appVersion: String? = null - var description: String? = null - chartYaml?.let { - appVersion = it[APP_VERSION] as String? - description = it[DESCRIPTION] as String? + try { + packageMetadataClient.saveMetadata(metadataSaveRequest) + } catch (ignore: Exception) { } - return Pair(appVersion, description) } + /** * 获取对应存储节点路径 * 特殊:manifest文件按tag存储, 但是查询时存在tag/digest */ override fun getNodeFullPath(artifactInfo: OciArtifactInfo): String? { - if (artifactInfo is OciManifestArtifactInfo) { - // 根据类型解析实际存储路径,manifest获取路径有tag/digest - if (artifactInfo.isValidDigest) { - return getNodeByDigest( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - digestStr = artifactInfo.reference - ).fullPath - } + if (artifactInfo is OciManifestArtifactInfo && artifactInfo.isValidDigest) { + // 根据类型解析实际存储路径,manifest获取路径有tag/digest, + // 当为digest 时,查当前仓库下,在package目录下,且sha256为digest的节点 + val path = OciLocationUtils.buildManifestFolderPath(artifactInfo.packageName) + return getNodeByDigest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + digestStr = artifactInfo.reference, + path = path + )?.fullPath } return artifactInfo.getArtifactFullPath() } @@ -920,23 +788,28 @@ class OciOperationServiceImpl( override fun getNodeByDigest( projectId: String, repoName: String, - digestStr: String - ): NodeProperty { + digestStr: String, + path: String? + ): NodeProperty? { val ociDigest = OciDigest(digestStr) val queryModel = NodeQueryBuilder() .select(NODE_FULL_PATH, MD5, OCI_NODE_SIZE) .projectId(projectId) .repoName(repoName) .sha256(ociDigest.getDigestHex()) - .sortByAsc(NODE_FULL_PATH) - val result = nodeClient.search(queryModel.build()).data ?: run { + .sortByAsc(NODE_FULL_PATH).apply { + path?.let { + this.path(path, OperationType.PREFIX) + } + } + val result = nodeClient.search(queryModel.build()).data + if (result == null || result.records.isEmpty()) { logger.warn( "Could not find $digestStr " + "in repo $projectId|$repoName" ) - return NodeProperty() + return null } - if (result.records.isEmpty()) return NodeProperty() return NodeProperty( fullPath = result.records[0][NODE_FULL_PATH] as String, md5 = result.records[0][MD5] as String?, @@ -951,25 +824,29 @@ class OciOperationServiceImpl( * 2 docker-local/nginx/_uploads/ 临时存储上传的blobs,待manifest文件上传成功后移到到对应版本下,如docker-local/nginx/latest */ override fun getDockerNode(artifactInfo: OciArtifactInfo): String? { - if (artifactInfo is OciManifestArtifactInfo) { - // 根据类型解析实际存储路径,manifest获取路径有tag/digest - if (artifactInfo.isValidDigest) + return when (artifactInfo) { + is OciManifestArtifactInfo -> { + // 根据类型解析实际存储路径,manifest获取路径有tag/digest + if (artifactInfo.isValidDigest){ + return getNodeByDigest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + digestStr = artifactInfo.reference, + path = "/${artifactInfo.packageName}/" + )?.fullPath + } + return "/${artifactInfo.packageName}/${artifactInfo.reference}/manifest.json" + } + is OciBlobArtifactInfo -> { + val digestStr = artifactInfo.digest ?: StringPool.EMPTY return getNodeByDigest( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, - digestStr = artifactInfo.reference - ).fullPath - return "/${artifactInfo.packageName}/${artifactInfo.reference}/manifest.json" - } - if (artifactInfo is OciBlobArtifactInfo) { - val digestStr = artifactInfo.digest ?: StringPool.EMPTY - return getNodeByDigest( - projectId = artifactInfo.projectId, - repoName = artifactInfo.repoName, - digestStr = digestStr - ).fullPath + digestStr = digestStr + )?.fullPath + } + else -> null } - return null } override fun getReturnDomain(request: HttpServletRequest): String { @@ -1093,6 +970,144 @@ class OciOperationServiceImpl( return OciTagResult(result.totalRecords, data) } + override fun getPackagesFromThirdPartyRepo(projectId: String, repoName: String) { + val repositoryDetail = repositoryClient.getRepoDetail(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") + buildImagePackagePullContext(projectId, repoName, repositoryDetail.configuration).forEach { + pluginManager.applyExtension { + queryAndCreateDockerPackageInfo(it) + } + } + } + + override fun deleteBlobsFolderAfterRefreshed( + projectId: String, repoName: String, pName: String, userId: String + ) { + repositoryClient.getRepoInfo(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") + val blobsFolderPath = buildBlobsFolderPath(pName) + val fullPaths = nodeClient.listNode( + projectId, repoName, blobsFolderPath, includeFolder = false, deep = false + ).data?.map { it.fullPath } + if (fullPaths.isNullOrEmpty()) return + logger.info("Blobs of package $pName in folder $blobsFolderPath will be deleted in $projectId|$repoName") + val request = NodesDeleteRequest( + projectId = projectId, + repoName = repoName, + fullPaths = fullPaths, + operator = userId + ) + nodeClient.deleteNodes(request) + } + + /** + * 调整blob目录 + * 从/packageName/blobs/xxx 到/packageName/blobs/version/xxx + */ + override fun refreshBlobNode( + projectId: String, + repoName: String, + pName: String, + pVersion: String, + userId: String + ): Boolean { + val repoInfo = repositoryClient.getRepoInfo(projectId, repoName).data + ?: throw RepoNotFoundException("$projectId|$repoName") + val packageName = PackageKeys.resolveName(repoInfo.type.name.toLowerCase(), pName) + val manifestPath = OciLocationUtils.buildManifestPath(packageName, pVersion) + logger.info("Manifest $manifestPath will be refreshed") + val ociArtifactInfo = OciManifestArtifactInfo( + projectId = repoInfo.projectId, + repoName = repoInfo.name, + packageName = packageName, + version = pVersion, + reference = pVersion, + isValidDigest = false + ) + val manifestNode = nodeClient.getNodeDetail( + repoInfo.projectId, repoInfo.name, manifestPath + ).data ?: run { + val oldDockerFullPath = getDockerNode(ociArtifactInfo) ?: return false + nodeClient.getNodeDetail( + repoInfo.projectId, repoInfo.name, oldDockerFullPath + ).data ?: return false + } + val refreshedMetadat = manifestNode.nodeMetadata.firstOrNull { it.key == BLOB_PATH_REFRESHED_KEY} + if (refreshedMetadat != null) { + logger.info("$manifestPath has been refreshed, ignore it") + return true + } + val storageCredentials = repoInfo.storageCredentialsKey?.let { storageCredentialsClient.findByKey(it).data } + val manifest = loadManifest( + manifestNode.sha256!!, manifestNode.size, storageCredentials + ) ?: run { + logger.warn("The content of manifest.json ${manifestNode.fullPath} is null, check the mediaType.") + return false + } + var refreshStatus = true + OciUtils.manifestIterator(manifest).forEach { + refreshStatus = refreshStatus && doSyncBlob(it, ociArtifactInfo, userId) + } + + if (refreshStatus) { + updateNodeMetaData( + projectId = repoInfo.projectId, + repoName = repoInfo.name, + fullPath = manifestPath, + metadata = mapOf(BLOB_PATH_REFRESHED_KEY to true) + ) + } + logger.info("The status of path $manifestPath refreshed is $refreshStatus") + return refreshStatus + } + + + + + + + private fun buildImagePackagePullContext( + projectId: String, + repoName: String, + config: RepositoryConfiguration + ): List { + val result = mutableListOf() + when (config) { + is RemoteConfiguration -> { + try { + val remoteUrl = UrlFormatter.addProtocol(config.url) + result.add(ImagePackagePullContext( + projectId = projectId, + repoName = repoName, + remoteUrl = remoteUrl, + userName = config.credentials.username, + password = config.credentials.password + )) + } catch (e: Exception) { + logger.warn("illegal remote url ${config.url} for repo $projectId|$repoName") + } + } + is CompositeConfiguration -> { + config.proxy.channelList.forEach { + try { + val remoteUrl = UrlFormatter.addProtocol(it.url) + result.add(ImagePackagePullContext( + projectId = projectId, + repoName = repoName, + remoteUrl = remoteUrl, + userName = it.username, + password = it.password + )) + } catch (e: Exception) { + logger.warn("illegal proxy url ${it.url} for repo $projectId|$repoName") + } + } + } + else -> throw UnsupportedOperationException() + } + return result + } + private fun buildString(stageTag: List): String { if (stageTag.isEmpty()) return StringPool.EMPTY return StringUtils.join(stageTag.toTypedArray()).toString() diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt index e5a9fd2bd9..c7a3c534d8 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/ObjectBuildUtils.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.artifact.constant.SOURCE_TYPE import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactChannel import com.tencent.bkrepo.common.artifact.util.PackageKeys import com.tencent.bkrepo.common.security.util.SecurityUtils +import com.tencent.bkrepo.oci.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.oci.constant.DIGEST_LIST import com.tencent.bkrepo.oci.constant.IMAGE_VERSION import com.tencent.bkrepo.oci.constant.MEDIA_TYPE @@ -76,7 +77,8 @@ object ObjectBuildUtils { fullPath: String, sha256: String, md5: String, - metadata: List? = null + metadata: List? = null, + userId: String = SecurityUtils.getUserId() ): NodeCreateRequest { return NodeCreateRequest( projectId = projectId, @@ -86,7 +88,7 @@ object ObjectBuildUtils { size = size, sha256 = sha256, md5 = md5, - operator = SecurityUtils.getUserId(), + operator = userId, overwrite = true, nodeMetadata = metadata ) @@ -95,19 +97,16 @@ object ObjectBuildUtils { fun buildMetadata( mediaType: String, version: String?, - yamlData: Map? = null, digestList: List? = null, sourceType: ArtifactChannel? = null ): MutableMap { return mutableMapOf( - MEDIA_TYPE to mediaType + MEDIA_TYPE to mediaType, + BLOB_PATH_REFRESHED_KEY to true ).apply { version?.let { this.put(IMAGE_VERSION, version) } digestList?.let { this.put(DIGEST_LIST, digestList) } sourceType?.let { this.put(SOURCE_TYPE, sourceType) } - yamlData?.let { - this.putAll(yamlData) - } } } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt index 674c1888c8..6cca3c70a4 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/util/OciResponseUtils.kt @@ -49,6 +49,7 @@ import com.tencent.bkrepo.oci.constant.HTTP_PROTOCOL_HTTP import com.tencent.bkrepo.oci.constant.HTTP_PROTOCOL_HTTPS import com.tencent.bkrepo.oci.constant.OCI_API_PREFIX import com.tencent.bkrepo.oci.pojo.digest.OciDigest +import com.tencent.bkrepo.oci.pojo.response.ResponseProperty import org.springframework.http.HttpHeaders import java.io.UnsupportedEncodingException import java.net.URI @@ -99,58 +100,11 @@ object OciResponseUtils { ) } - fun buildUploadResponse(domain: String, digest: OciDigest, locationStr: String, response: HttpServletResponse) { + fun buildUploadResponse(domain: String, responseProperty: ResponseProperty, response: HttpServletResponse) { uploadResponse( domain = domain, response = response, - status = HttpStatus.CREATED, - locationStr = locationStr, - digest = digest.toString(), - contentLength = 0 - ) - } - - fun buildBlobUploadUUIDResponse(domain: String, uuid: String, locationStr: String, response: HttpServletResponse) { - uploadResponse( - domain = domain, - response = response, - status = HttpStatus.ACCEPTED, - locationStr = locationStr, - uuid = uuid, - contentLength = 0 - ) - } - - fun buildBlobUploadPatchResponse( - domain: String, - uuid: String, - locationStr: String, - status: HttpStatus = HttpStatus.ACCEPTED, - response: HttpServletResponse, - range: Long - ) { - uploadResponse( - domain = domain, - response = response, - status = status, - locationStr = locationStr, - uuid = uuid, - contentLength = 0, - range = range - ) - } - - fun buildBlobMountResponse( - domain: String, - locationStr: String, - status: HttpStatus = HttpStatus.ACCEPTED, - response: HttpServletResponse - ) { - uploadResponse( - domain = domain, - response = response, - status = status, - locationStr = locationStr + responseProperty = responseProperty ) } @@ -175,29 +129,26 @@ object OciResponseUtils { private fun uploadResponse( domain: String, response: HttpServletResponse = HttpContextHolder.getResponse(), - status: HttpStatus, - locationStr: String, - digest: String? = null, - uuid: String? = null, - range: Long? = null, - contentLength: Int? = null + responseProperty: ResponseProperty ) { - val location = getResponseLocationURI(locationStr, domain) - response.status = status.value - response.addHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION) - digest?.let { - response.addHeader(DOCKER_CONTENT_DIGEST, digest) - } - response.addHeader(HttpHeaders.LOCATION, location) - uuid?.let { - response.addHeader(BLOB_UPLOAD_SESSION_ID, uuid) - response.addHeader(DOCKER_UPLOAD_UUID, uuid) - } - contentLength?.let { - response.addHeader(CONTENT_LENGTH, contentLength.toString()) - } - range?.let { - response.addHeader(RANGE, "0-${range - 1}") + with(responseProperty) { + val location = getResponseLocationURI(location!!, domain) + response.status = status!!.value + response.addHeader(DOCKER_HEADER_API_VERSION, DOCKER_API_VERSION) + digest?.let { + response.addHeader(DOCKER_CONTENT_DIGEST, digest.toString()) + } + response.addHeader(HttpHeaders.LOCATION, location) + uuid?.let { + response.addHeader(BLOB_UPLOAD_SESSION_ID, uuid) + response.addHeader(DOCKER_UPLOAD_UUID, uuid) + } + contentLength?.let { + response.addHeader(CONTENT_LENGTH, contentLength.toString()) + } + range?.let { + response.addHeader(RANGE, "0-${range!! - 1}") + } } } diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties index eb6c359888..1df4423048 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_en.properties @@ -36,4 +36,7 @@ oci.repo.not.found=Oci repository [{0}] not found! oci.delete.rules=Delete the blob [{0}] identified by digest! oci.version.not.found=The version [{0}] of artifact not found in repo [{1}] ! oci.manifest.invalid=Manifest invalid! -oci.digest.invalid=Digest invalid [{0}]! \ No newline at end of file +oci.digest.invalid=Digest invalid [{0}]! +oci.manifest.schema1.not.support=Schema1 of manifest not supported! +oci.remote.configuration.error= Remote configuration error, error is [{0}]! +oci.remote.credentials.invalid= Remote credentials invalid, error is [{0}]! \ No newline at end of file diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties index ba00b5bcdb..9e5117b24c 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_CN.properties @@ -36,4 +36,7 @@ oci.repo.not.found=Oci仓库 [{0}] 不存在! oci.delete.rules=制品 [{0}] 只能使用digest删除! oci.version.not.found=制品版本 [{0}] 在仓库 [{1}] 中不存在! oci.manifest.invalid=请检查Manifest内容是否符合规范! -oci.digest.invalid=上传内容Digest与提供的digest不一致 [{0}]! \ No newline at end of file +oci.digest.invalid=上传内容Digest与提供的digest不一致 [{0}]! +oci.manifest.schema1.not.support=Manifest文件不支持Schema1版本! +oci.remote.configuration.error= 请检查远程仓库配置信息, 错误:[{0}]! +oci.remote.credentials.invalid= 代理鉴权信息有误: [{0}],请确认! \ No newline at end of file diff --git a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties index 59782463fa..c9599531b3 100644 --- a/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties +++ b/src/backend/oci/biz-oci/src/main/resources/i18n/messages_zh_TW.properties @@ -36,4 +36,7 @@ oci.repo.not.found=Oci倉庫 [{0}] 不存在! oci.delete.rules=製品 [{0}] 只能digest刪除! oci.version.not.found=製品版本 [{0}] 在倉庫 [{1}] 中不存在! oci.manifest.invalid=請檢查Manifest內容是否符合規範! -oci.digest.invalid=上傳內容Digest與提供的digest不一致 [{0}]! \ No newline at end of file +oci.digest.invalid=上傳內容Digest與提供的digest不一致 [{0}]! +oci.manifest.schema1.not.support=Manifest文件不支持Schema1版本! +oci.remote.configuration.error= 請檢查遠程倉庫配置信息, 錯誤:[{0}]! +oci.remote.credentials.invalid= 代理鑒權信息有誤: [{0}] ,請確認! \ No newline at end of file diff --git a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt index f98f68e889..cb08bd2168 100644 --- a/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt +++ b/src/backend/replication/api-replication/src/main/kotlin/com/tencent/bkrepo/replication/constant/Constants.kt @@ -29,9 +29,6 @@ package com.tencent.bkrepo.replication.constant const val DEFAULT_VERSION = "1.0.0" -const val BEARER_REALM = "Bearer realm" -const val SERVICE = "service" -const val SCOPE = "scope" const val REPOSITORY = "repository" const val URL = "url" @@ -43,6 +40,8 @@ const val DOCKER_MANIFEST_JSON_FULL_PATH = "/%s/%s/manifest.json" const val DOCKER_LAYER_FULL_PATH = "/%s/%s/%s" const val OCI_MANIFEST_JSON_FULL_PATH = "/%s/manifest/%s/manifest.json" const val OCI_LAYER_FULL_PATH = "/%s/blobs/%s" +const val OCI_LAYER_FULL_PATH_V1 = "/%s/blobs/%s/%s" +const val BLOB_PATH_REFRESHED_KEY = "blobPathRefreshed" const val NODE_FULL_PATH = "fullPath" const val SIZE = "size" const val REPOSITORY_INFO = "repo" diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt index dd029ce030..ee395dce11 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/ArtifactReplicationHandler.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.constant.CHUNKED_UPLOAD import com.tencent.bkrepo.replication.constant.REPOSITORY_INFO @@ -44,7 +45,6 @@ import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.pojo.request.ReplicaType import com.tencent.bkrepo.replication.replica.base.context.FilePushContext import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.util.HttpUtils import com.tencent.bkrepo.replication.util.StreamRequestBody import okhttp3.Headers import okhttp3.MediaType.Companion.toMediaTypeOrNull @@ -352,7 +352,7 @@ abstract class ArtifactReplicationHandler( } catch (e: Exception) { val baseUrl = URL(url) val host = URL(baseUrl.protocol, baseUrl.host, StringPool.EMPTY).toString() - HttpUtils.buildUrl(host, location.removePrefix("/")) + UrlFormatter.buildUrl(host, location.removePrefix("/")) } } } @@ -368,7 +368,7 @@ abstract class ArtifactReplicationHandler( ): String { val baseUrl = URL(url) val suffixUrl = URL(baseUrl, baseUrl.path).toString() - return HttpUtils.buildUrl(suffixUrl, path, params) + return UrlFormatter.buildUrl(suffixUrl, path, params) } open fun buildRequestTag( diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt index 9b96f26361..8dec006a3b 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/handler/RemoteClusterArtifactReplicationHandler.kt @@ -28,13 +28,13 @@ package com.tencent.bkrepo.replication.replica.base.handler import com.tencent.bkrepo.common.api.constant.HttpStatus +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.constant.OCI_BLOBS_UPLOAD_FIRST_STEP_URL import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult import com.tencent.bkrepo.replication.replica.base.context.FilePushContext import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext -import com.tencent.bkrepo.replication.util.HttpUtils import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.net.URL @@ -105,7 +105,7 @@ class RemoteClusterArtifactReplicationHandler( ): String { val baseUrl = URL(url) val suffixUrl = URL(baseUrl, "/v2" + baseUrl.path).toString() - return HttpUtils.buildUrl(suffixUrl, path, params) + return UrlFormatter.buildUrl(suffixUrl, path, params) } companion object { private val logger = LoggerFactory.getLogger(RemoteClusterArtifactReplicationHandler::class.java) diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt index 5df18e0027..bfb0450916 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/internal/type/DockerPackageNodeMapper.kt @@ -4,13 +4,16 @@ import com.tencent.bkrepo.common.artifact.exception.ArtifactNotFoundException import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.storage.core.StorageService +import com.tencent.bkrepo.replication.constant.BLOB_PATH_REFRESHED_KEY import com.tencent.bkrepo.replication.constant.DOCKER_LAYER_FULL_PATH import com.tencent.bkrepo.replication.constant.DOCKER_MANIFEST_JSON_FULL_PATH import com.tencent.bkrepo.replication.constant.OCI_LAYER_FULL_PATH +import com.tencent.bkrepo.replication.constant.OCI_LAYER_FULL_PATH_V1 import com.tencent.bkrepo.replication.constant.OCI_MANIFEST_JSON_FULL_PATH import com.tencent.bkrepo.replication.util.ManifestParser import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.api.RepositoryClient +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.packages.PackageSummary import com.tencent.bkrepo.repository.pojo.packages.PackageVersion import org.springframework.stereotype.Component @@ -61,15 +64,40 @@ class DockerPackageNodeMapper( throw ArtifactNotFoundException("Could not read manifest.json, $e") } manifestInfo!!.descriptors?.forEach { - val replace = it.replace(":", "__") - if (isOci) { - result.add(OCI_LAYER_FULL_PATH.format(name, replace)) - } else { - result.add(DOCKER_LAYER_FULL_PATH.format(name, version, replace)) - } + result.add(buildBlobPath( + descriptor = it, + packageName = name, + version = version, + isOci = isOci, + nodeDetail = nodeDetail + )) } result.add(manifestFullPath) return result } } + + /** + * 通过包名、版本、sha256拼接出blob路径 + */ + private fun buildBlobPath( + descriptor: String, + packageName: String, + version: String, + isOci: Boolean, + nodeDetail: NodeDetail + ): String { + val replace = descriptor.replace(":", "__") + return if (isOci) { + // 镜像blob路径格式有调整,从/package/blobs/下调至//package/blobs/version/ + val refreshedMetadata = nodeDetail.nodeMetadata.firstOrNull { it.key == BLOB_PATH_REFRESHED_KEY} + if (refreshedMetadata != null) { + OCI_LAYER_FULL_PATH_V1.format(packageName, version, replace) + } else { + OCI_LAYER_FULL_PATH.format(packageName, replace) + } + } else { + DOCKER_LAYER_FULL_PATH.format(packageName, version, replace) + } + } } diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt index 3639472d13..2e57791dc0 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/helm/HelmArtifactPushClient.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.util.BasicAuthUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.service.cluster.ClusterInfo import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager @@ -40,7 +41,6 @@ import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext import com.tencent.bkrepo.replication.replica.base.handler.DefaultHandler import com.tencent.bkrepo.replication.replica.base.impl.remote.base.PushClient -import com.tencent.bkrepo.replication.util.HttpUtils import com.tencent.bkrepo.repository.pojo.node.NodeDetail import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.MultipartBody @@ -258,7 +258,7 @@ class HelmArtifactPushClient( ): String { val baseUrl = URL(url) val v2Url = URL(baseUrl, baseUrl.path) - return HttpUtils.buildUrl(v2Url.toString(), path, params) + return UrlFormatter.buildUrl(v2Url.toString(), path, params) } private fun buildAuthRequestProperties(clusterInfo: ClusterInfo): RequestProperty { diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt index 2a7ca766f8..989ac50cf1 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/base/impl/remote/type/oci/OciAuthorizationService.kt @@ -31,14 +31,11 @@ import com.tencent.bkrepo.common.api.constant.BEARER_AUTH_PREFIX import com.tencent.bkrepo.common.api.constant.HttpHeaders import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.api.constant.StringPool -import com.tencent.bkrepo.common.api.constant.StringPool.QUESTION +import com.tencent.bkrepo.common.api.util.AuthenticationUtil.buildAuthenticationUrl +import com.tencent.bkrepo.common.api.util.AuthenticationUtil.parseWWWAuthenticateHeader import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.api.util.toJsonString -import com.tencent.bkrepo.replication.constant.BEARER_REALM -import com.tencent.bkrepo.replication.constant.SCOPE -import com.tencent.bkrepo.replication.constant.SERVICE import com.tencent.bkrepo.replication.pojo.docker.OciResponse -import com.tencent.bkrepo.replication.pojo.remote.AuthenticationProperty import com.tencent.bkrepo.replication.pojo.remote.BearerToken import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import com.tencent.bkrepo.replication.replica.base.impl.remote.base.AuthorizationService @@ -106,10 +103,14 @@ class OciAuthorizationService : AuthorizationService { val httpRequest = HttpUtils.wrapperRequest(property) httpClient.newCall(httpRequest).execute().use { if (!it.isSuccessful) { - val error = JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java) + val error = try { + JsonUtils.objectMapper.readValue(it.body!!.byteStream(), OciResponse::class.java).toJsonString() + } catch (ignore: Exception) { + StringPool.EMPTY + } throw ArtifactPushException( "Could not get token from auth service," + - " code is ${it.code} and response is ${error.toJsonString()}" + " code is ${it.code} and response is $error" ) } try { @@ -123,44 +124,6 @@ class OciAuthorizationService : AuthorizationService { } } - /** - * 解析返回头中的WWW_AUTHENTICATE字段, 只针对为Bearer realm - */ - private fun parseWWWAuthenticateHeader(wwwAuthenticate: String, scope: String?): AuthenticationProperty? { - val map: MutableMap = mutableMapOf() - return try { - val params = wwwAuthenticate.split("\",") - params.forEach { - val param = it.split(Regex("="),2) - val name = param.first() - val value = param.last().replace("\"", "") - map[name] = value - } - AuthenticationProperty( - authUrl = map[BEARER_REALM]!!, - service = map[SERVICE]!!, - scope = scope - ) - } catch (e: Exception) { - logger.warn("Parsing wwwAuthenticate header error: ${e.message}") - null - } - } - - private fun buildAuthenticationUrl(property: AuthenticationProperty, userName: String?): String? { - if (property.authUrl.isBlank()) return null - var result = if (property.authUrl.contains(QUESTION)) { - "${property.authUrl}&$SERVICE=${property.service}" - } else { - "${property.authUrl}?$SERVICE=${property.service}" - } - property.scope?.let { - result += "&$SCOPE=${property.scope}" - } - userName?.let { result += "&account=$userName" } - return result - } - companion object { private val logger = LoggerFactory.getLogger(OciAuthorizationService::class.java) } diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt index 082ad9fdbb..4cb6ffba58 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/service/impl/RemoteNodeServiceImpl.kt @@ -33,6 +33,7 @@ import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.artifact.event.packages.VersionCreatedEvent import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.util.PackageKeys +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter.addProtocol import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace import com.tencent.bkrepo.replication.api.ReplicaTaskOperationClient @@ -69,7 +70,6 @@ import com.tencent.bkrepo.replication.service.RemoteNodeService import com.tencent.bkrepo.replication.service.ReplicaNodeDispatchService import com.tencent.bkrepo.replication.service.ReplicaRecordService import com.tencent.bkrepo.replication.service.ReplicaTaskService -import com.tencent.bkrepo.replication.util.HttpUtils.addProtocol import com.tencent.bkrepo.replication.util.ReplicationMetricsRecordUtil.convertToReplicationTaskMetricsRecord import com.tencent.bkrepo.replication.util.ReplicationMetricsRecordUtil.toJson import org.slf4j.LoggerFactory diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt index 914f02e927..1010c53c09 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/util/HttpUtils.kt @@ -27,19 +27,14 @@ package com.tencent.bkrepo.replication.util -import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.api.constant.HttpHeaders -import com.tencent.bkrepo.common.api.constant.StringPool -import com.tencent.bkrepo.common.storage.innercos.retry -import com.tencent.bkrepo.replication.constant.DELAY_IN_SECONDS -import com.tencent.bkrepo.replication.constant.RETRY_COUNT +import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter.addParams import com.tencent.bkrepo.replication.pojo.blob.RequestTag import com.tencent.bkrepo.replication.pojo.remote.RequestProperty import okhttp3.Request import org.springframework.web.bind.annotation.RequestMethod import java.io.IOException import java.net.HttpURLConnection -import java.net.MalformedURLException import java.net.URL object HttpUtils { @@ -51,7 +46,7 @@ object HttpUtils { ): Request { with(requestProperty) { val url = params?.let { - buildUrl(url = requestUrl!!, params = params!!) + addParams(url = requestUrl!!, params = params!!) } ?: requestUrl!! var builder = Request.Builder() .url(url) @@ -73,46 +68,6 @@ object HttpUtils { } } - /** - * 拼接url - */ - fun buildUrl( - url: String, - path: String = StringPool.EMPTY, - params: String = StringPool.EMPTY, - ): String { - val builder = StringBuilder(url) - if (path.isNotBlank()) { - builder.append(CharPool.SLASH).append(path.trimStart(CharPool.SLASH)) - } - if (params.isNotBlank()) { - if (builder.contains(CharPool.QUESTION)) { - builder.append(CharPool.AND).append(params) - } else { - builder.append(CharPool.QUESTION).append(params) - } - } - return builder.toString() - } - - /** - * 针对url如果没传protocol, 则默认以https请求发送 - */ - fun addProtocol(registry: String): URL { - try { - return URL(registry) - } catch (ignore: MalformedURLException) { - } - val url = URL("${StringPool.HTTPS}$registry") - return try { - retry(times = RETRY_COUNT, delayInSeconds = DELAY_IN_SECONDS) { - validateHttpsProtocol(url) - url - } - } catch (ignore: Exception) { - URL(url.toString().replaceFirst("^https".toRegex(), "http")) - } - } /** * Pings a HTTP URL. This effectively sends a HEAD request and returns `true` if the response code is in @@ -139,21 +94,6 @@ object HttpUtils { } } - /** - * 验证registry是否支持https - */ - private fun validateHttpsProtocol(url: URL): Boolean { - return try { - val http: HttpURLConnection = url.openConnection() as HttpURLConnection - http.instanceFollowRedirects = false - http.responseCode - http.disconnect() - true - } catch (e: Exception) { - throw e - } - } - /** * 从Content-Range头中解析出起始位置 */ diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt index d8fabdba8d..45afee4fcc 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/NodeClient.kt @@ -133,7 +133,7 @@ interface NodeClient { @ApiOperation("删除节点") @DeleteMapping("/batch/delete") - fun deleteNodes(nodesDeleteRequest: NodesDeleteRequest): Response + fun deleteNodes(@RequestBody nodesDeleteRequest: NodesDeleteRequest): Response @ApiOperation("恢复节点") @PostMapping("/restore") @@ -194,4 +194,14 @@ interface NodeClient { @PathVariable repoName: String, @RequestParam fullPath: String ): Response> + + @ApiOperation("通过sha256查询已删除节点") + @GetMapping("/deletedBySha256/detail/{projectId}/{repoName}") + fun getDeletedNodeDetailBySha256( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam sha256: String + ): Response + + } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt index f6c3c79e34..c954cf223c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/NodeController.kt @@ -178,4 +178,10 @@ class NodeController( val artifactInfo = DefaultArtifactInfo(projectId, repoName, fullPath) return ResponseBuilder.success(nodeService.getDeletedNodeDetail(artifactInfo)) } + + override fun getDeletedNodeDetailBySha256( + projectId: String, repoName: String, sha256: String + ): Response { + return ResponseBuilder.success(nodeService.getDeletedNodeDetailBySha256(projectId, repoName, sha256)) + } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt index 2fa41c70bb..617603e44e 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreOperation.kt @@ -48,6 +48,10 @@ interface NodeRestoreOperation { */ fun getDeletedNodeDetail(artifact: ArtifactInfo): List + /** + * 根据 sha256 查询被删除的节点详情 + */ + fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? /** * 恢复被删除的节点 * @return 恢复节点数量 diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt index 4cfe17bc50..b3eb0d925c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt @@ -50,6 +50,7 @@ import com.tencent.bkrepo.repository.util.MetadataUtils import com.tencent.bkrepo.repository.util.NodeQueryHelper import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedFolderQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointListQuery +import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointListQueryBySha256 import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeDeletedPointQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeListQuery import com.tencent.bkrepo.repository.util.NodeQueryHelper.nodeQuery @@ -76,6 +77,12 @@ open class NodeRestoreSupport( } } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + val query = nodeDeletedPointListQueryBySha256(projectId, repoName, sha256) + val node = nodeDao.findOne(query) + return convertToDetail(node) + } + override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { with(resolveContext(artifact, nodeRestoreOption)) { return restoreNode(this) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt index 268d705553..1f3fb9ac84 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt @@ -149,6 +149,12 @@ class NodeServiceImpl( return NodeRestoreSupport(this).getDeletedNodeDetail(artifact) } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + return NodeRestoreSupport(this).getDeletedNodeDetailBySha256( + projectId, repoName, sha256 + ) + } + @Transactional(rollbackFor = [Throwable::class]) override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { return NodeRestoreSupport(this).restoreNode(artifact, nodeRestoreOption) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt index 614a9ea66d..34479b7496 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -162,6 +162,10 @@ class EdgeNodeServiceImpl( return NodeRestoreSupport(this).getDeletedNodeDetail(artifact) } + override fun getDeletedNodeDetailBySha256(projectId: String, repoName: String, sha256: String): NodeDetail? { + return NodeRestoreSupport(this).getDeletedNodeDetailBySha256(projectId, repoName, sha256) + } + @Transactional(rollbackFor = [Throwable::class]) override fun restoreNode(artifact: ArtifactInfo, nodeRestoreOption: NodeRestoreOption): NodeRestoreResult { centerNodeClient.restoreNode(NodeRestoreRequest(artifact, nodeRestoreOption)) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt index a489df1bb6..e16a1ca508 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt @@ -128,6 +128,17 @@ object NodeQueryHelper { return Query(criteria).with(Sort.by(Sort.Direction.DESC, TNode::deleted.name)) } + /** + * 通过sha256查询被删除节点详情 + */ + fun nodeDeletedPointListQueryBySha256(projectId: String, repoName: String, sha256: String): Query { + val criteria = where(TNode::projectId).isEqualTo(projectId) + .and(TNode::repoName).isEqualTo(repoName) + .and(TNode::sha256).isEqualTo(sha256) + .and(TNode::deleted).ne(null) + return Query(criteria).with(Sort.by(Sort.Direction.DESC, TNode::deleted.name)) + } + /** * 查询单个被删除节点 */