generic类型制品支持S3协议获取 TencentBlueKing#1296 (TencentBlueKing#1457)

* generic类型制品支持S3协议获取 TencentBlueKing#1296 * generic类型制品支持S3协议获取 TencentBlueKing#1296 * generic类型制品支持S3协议获取 TencentBlueKing#1296 * generic类型制品支持S3协议获取 TencentBlueKing#1296 * generic类型制品支持S3协议获取 TencentBlueKing#1296 * generic类型制品支持S3协议获取 TencentBlueKing#1296 * generic类型制品支持S3协议获取 TencentBlueKing#1296
owenlxu · Dec 28, 2023 · ff93cd9 · ff93cd9
1 parent 7ed2fe8
commit ff93cd9
Show file tree

Hide file tree

Showing 53 changed files with 2,315 additions and 78 deletions.
diff --git a/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/api/ServiceUserClient.kt b/src/backend/auth/api-auth/src/main/kotlin/com/tencent/bkrepo/auth/api/ServiceUserClient.kt
@@ -97,4 +97,10 @@ interface ServiceUserClient {
     fun userInfoById(
         @PathVariable uid: String
     ): Response<UserInfo?>
+
+    @ApiOperation("获取用户pwd ")
+    @GetMapping("/userpwd/{uid}")
+    fun userPwdById(
+        @PathVariable uid: String
+    ): Response<String?>
 }
diff --git a/...-auth/src/main/kotlin/com/tencent/bkrepo/auth/controller/service/ServiceUserController.kt b/...-auth/src/main/kotlin/com/tencent/bkrepo/auth/controller/service/ServiceUserController.kt
@@ -85,4 +85,8 @@ class ServiceUserController @Autowired constructor(
     override fun userInfoById(uid: String): Response<UserInfo?> {
         return ResponseBuilder.success(userService.getUserInfoById(uid))
     }
+
+    override fun userPwdById(uid: String): Response<String?> {
+        return ResponseBuilder.success(userService.getUserPwdById(uid))
+    }
 }
diff --git a/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/UserService.kt b/src/backend/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/UserService.kt
@@ -79,6 +79,8 @@ interface UserService {
 
     fun getUserInfoById(userId: String): UserInfo?
 
+    fun getUserPwdById(userId: String): String?
+
     fun updatePassword(userId: String, oldPwd: String, newPwd: String): Boolean
 
     fun resetPassword(userId: String): Boolean

diff --git a/...nd/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/UserServiceImpl.kt b/...nd/auth/biz-auth/src/main/kotlin/com/tencent/bkrepo/auth/service/local/UserServiceImpl.kt
@@ -371,6 +371,11 @@ class UserServiceImpl constructor(
         return UserRequestUtil.convToUserInfo(tUser)
     }
 
+    override fun getUserPwdById(userId: String): String? {
+        val tUser = userRepository.findFirstByUserId(userId) ?: return null
+        return tUser.pwd
+    }
+
     override fun updatePassword(userId: String, oldPwd: String, newPwd: String): Boolean {
         val query = UserQueryHelper.getUserByIdAndPwd(userId, oldPwd)
         val user = mongoTemplate.find(query, TUser::class.java)

diff --git a/src/backend/buildSrc/src/main/kotlin/Versions.kt b/src/backend/buildSrc/src/main/kotlin/Versions.kt
@@ -66,4 +66,5 @@ object Versions {
     const val Jasypt = "3.0.5"
     const val CryptoJavaSdk = "1.1.0"
     const val IamJavaSdk = "1.0.30-SNAPSHOT"
+    const val Dom4j = "2.1.0"
 }
diff --git a/...mon/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt b/...mon/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/message/CommonMessageCode.kt
@@ -60,7 +60,6 @@ enum class CommonMessageCode(private val key: String) : MessageCode {
     OPERATION_CROSS_CLUSTER_NOT_ALLOWED("operation.cross-cluster.not-allowed"),
     MEDIA_TYPE_UNACCEPTABLE("system.media-type.unacceptable"),
     TOO_MANY_REQUESTS("too.many.requests"),
-
     ;
 
     override fun getBusinessCode() = ordinal + 1

diff --git a/...ct/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryType.kt b/...ct/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/pojo/RepositoryType.kt
@@ -52,6 +52,7 @@ enum class RepositoryType {
     LFS,
     DDC,
     SVN,
+    S3
     ;
 
     companion object {

diff --git a/...in/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt b/...in/com/tencent/bkrepo/common/artifact/resolve/response/AbstractArtifactResourceHandler.kt
@@ -0,0 +1,125 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.artifact.resolve.response
+
+import com.tencent.bkrepo.common.api.constant.HttpStatus
+import com.tencent.bkrepo.common.api.exception.TooManyRequestsException
+import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException
+import com.tencent.bkrepo.common.artifact.metrics.RecordAbleInputStream
+import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
+import com.tencent.bkrepo.common.artifact.stream.STREAM_BUFFER_SIZE
+import com.tencent.bkrepo.common.artifact.stream.rateLimit
+import com.tencent.bkrepo.common.artifact.util.http.IOExceptionUtils
+import com.tencent.bkrepo.common.storage.core.StorageProperties
+import com.tencent.bkrepo.common.storage.monitor.Throughput
+import com.tencent.bkrepo.common.storage.monitor.measureThroughput
+import org.springframework.http.HttpMethod
+import org.springframework.util.unit.DataSize
+import java.io.IOException
+import javax.servlet.http.HttpServletRequest
+import javax.servlet.http.HttpServletResponse
+
+
+abstract class AbstractArtifactResourceHandler(
+    private val storageProperties: StorageProperties
+) : ArtifactResourceWriter {
+    /**
+     * 获取动态buffer size
+     * @param totalSize 数据总大小
+     */
+    protected  fun getBufferSize(totalSize: Int): Int {
+        val bufferSize = storageProperties.response.bufferSize.toBytes().toInt()
+        return when {
+            bufferSize < 0 || totalSize < 0 -> STREAM_BUFFER_SIZE
+            totalSize < bufferSize -> totalSize
+            else -> bufferSize
+        }
+    }
+
+    /**
+     * 将仓库级别的限速配置导入
+     * 当同时存在全局限速配置以及仓库级别限速配置时，以仓库级别配置优先
+     */
+    protected fun responseRateLimitWrapper(rateLimit: DataSize): Long {
+        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
+        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1)) {
+            return rateLimitOfRepo.responseRateLimit.toBytes()
+        }
+        return rateLimit.toBytes()
+    }
+
+    /**
+     * 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接
+     */
+    protected fun responseRateLimitCheck() {
+        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
+        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1) &&
+            rateLimitOfRepo.responseRateLimit <= storageProperties.response.circuitBreakerThreshold) {
+            throw TooManyRequestsException(
+                "The circuit breaker is activated when too many download requests are made to the service!"
+            )
+        }
+    }
+
+    /**
+     * 将数据流以Range方式写入响应
+     */
+    protected fun writeRangeStream(
+        resource: ArtifactResource,
+        request: HttpServletRequest,
+        response: HttpServletResponse
+    ): Throughput {
+        val inputStream = resource.getSingleStream()
+        if (request.method == HttpMethod.HEAD.name) {
+            return Throughput.EMPTY
+        }
+        val recordAbleInputStream = RecordAbleInputStream(inputStream)
+        try {
+            return measureThroughput {
+                recordAbleInputStream.rateLimit(responseRateLimitWrapper(storageProperties.response.rateLimit)).use {
+                    it.copyTo(
+                        out = response.outputStream,
+                        bufferSize = getBufferSize(inputStream.range.length.toInt())
+                    )
+                }
+            }
+        } catch (exception: IOException) {
+            // 直接向上抛IOException经过CglibAopProxy会抛java.lang.reflect.UndeclaredThrowableException: null
+            // 由于已经设置了Content-Type为application/octet-stream, spring找不到对应的Converter，导致抛
+            // org.springframework.http.converter.HttpMessageNotWritableException异常，会重定向到/error页面
+            // 又因为/error页面不存在，最终返回404，所以要对IOException进行包装，在上一层捕捉处理
+            val message = exception.message.orEmpty()
+            val status = if (IOExceptionUtils.isClientBroken(exception))
+                HttpStatus.BAD_REQUEST
+            else
+                HttpStatus.INTERNAL_SERVER_ERROR
+            throw ArtifactResponseException(message, status)
+        }
+    }
+
+}
diff --git a/...tlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt b/...tlin/com/tencent/bkrepo/common/artifact/resolve/response/DefaultArtifactResourceWriter.kt
@@ -31,16 +31,13 @@ import com.tencent.bkrepo.common.api.constant.HttpHeaders
 import com.tencent.bkrepo.common.api.constant.HttpStatus
 import com.tencent.bkrepo.common.api.constant.MediaTypes
 import com.tencent.bkrepo.common.api.constant.StringPool
-import com.tencent.bkrepo.common.api.exception.TooManyRequestsException
 import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_MD5
 import com.tencent.bkrepo.common.artifact.constant.X_CHECKSUM_SHA256
 import com.tencent.bkrepo.common.artifact.exception.ArtifactResponseException
 import com.tencent.bkrepo.common.artifact.metrics.RecordAbleInputStream
 import com.tencent.bkrepo.common.artifact.path.PathUtils
-import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
 import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
 import com.tencent.bkrepo.common.artifact.stream.Range
-import com.tencent.bkrepo.common.artifact.stream.STREAM_BUFFER_SIZE
 import com.tencent.bkrepo.common.artifact.stream.closeQuietly
 import com.tencent.bkrepo.common.artifact.stream.rateLimit
 import com.tencent.bkrepo.common.artifact.util.http.HttpHeaderUtils.determineMediaType
@@ -52,7 +49,6 @@ import com.tencent.bkrepo.common.storage.monitor.Throughput
 import com.tencent.bkrepo.common.storage.monitor.measureThroughput
 import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import org.springframework.http.HttpMethod
-import org.springframework.util.unit.DataSize
 import java.io.IOException
 import java.time.LocalDateTime
 import java.time.ZoneOffset
@@ -67,7 +63,7 @@ import javax.servlet.http.HttpServletResponse
  */
 open class DefaultArtifactResourceWriter(
     private val storageProperties: StorageProperties
-) : ArtifactResourceWriter {
+) : AbstractArtifactResourceHandler(storageProperties) {
 
     @Throws(ArtifactResponseException::class)
     override fun write(resource: ArtifactResource): Throughput {
@@ -174,39 +170,6 @@ open class DefaultArtifactResourceWriter(
         return localDateTime.atZone(ZoneOffset.systemDefault()).toInstant().toEpochMilli()
     }
 
-    /**
-     * 将数据流以Range方式写入响应
-     */
-    private fun writeRangeStream(
-        resource: ArtifactResource,
-        request: HttpServletRequest,
-        response: HttpServletResponse
-    ): Throughput {
-        val inputStream = resource.getSingleStream()
-        if (request.method == HttpMethod.HEAD.name) {
-            return Throughput.EMPTY
-        }
-        val recordAbleInputStream = RecordAbleInputStream(inputStream)
-        try {
-            return measureThroughput {
-                recordAbleInputStream.rateLimit(responseRateLimitWrapper(storageProperties.response.rateLimit)).use {
-                    it.copyTo(
-                        out = response.outputStream,
-                        bufferSize = getBufferSize(inputStream.range.length.toInt())
-                    )
-                }
-            }
-        } catch (exception: IOException) {
-            // 直接向上抛IOException经过CglibAopProxy会抛java.lang.reflect.UndeclaredThrowableException: null
-            // 由于已经设置了Content-Type为application/octet-stream, spring找不到对应的Converter，导致抛
-            // org.springframework.http.converter.HttpMessageNotWritableException异常，会重定向到/error页面
-            // 又因为/error页面不存在，最终返回404，所以要对IOException进行包装，在上一层捕捉处理
-            val message = exception.message.orEmpty()
-            val status = if (isClientBroken(exception)) HttpStatus.BAD_REQUEST else HttpStatus.INTERNAL_SERVER_ERROR
-            throw ArtifactResponseException(message, status)
-        }
-    }
-
     /**
      * 将数据流以ZipOutputStream方式写入响应
      */
@@ -248,31 +211,6 @@ open class DefaultArtifactResourceWriter(
         }
     }
 
-    /**
-     * 将仓库级别的限速配置导入
-     * 当同时存在全局限速配置以及仓库级别限速配置时，以仓库级别配置优先
-     */
-    private fun responseRateLimitWrapper(rateLimit: DataSize): Long {
-        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
-        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1)) {
-            return rateLimitOfRepo.responseRateLimit.toBytes()
-        }
-        return rateLimit.toBytes()
-    }
-
-    /**
-     * 当仓库配置下载限速小于等于最低限速时则直接将请求断开, 避免占用过多连接
-     */
-    private fun responseRateLimitCheck() {
-        val rateLimitOfRepo = ArtifactContextHolder.getRateLimitOfRepo()
-        if (rateLimitOfRepo.responseRateLimit != DataSize.ofBytes(-1) &&
-            rateLimitOfRepo.responseRateLimit <= storageProperties.response.circuitBreakerThreshold) {
-            throw TooManyRequestsException(
-                "The circuit breaker is activated when too many download requests are made to the service!"
-            )
-        }
-    }
-
     /**
      * 判断charset,一些媒体类型设置了charset会影响其表现，如application/vnd.android.package-archive
      * */
@@ -290,18 +228,6 @@ open class DefaultArtifactResourceWriter(
         return node.sha256!!
     }
 
-    /**
-     * 获取动态buffer size
-     * @param totalSize 数据总大小
-     */
-    private fun getBufferSize(totalSize: Int): Int {
-        val bufferSize = storageProperties.response.bufferSize.toBytes().toInt()
-        if (bufferSize < 0 || totalSize < 0) {
-            return STREAM_BUFFER_SIZE
-        }
-        return if (totalSize < bufferSize) totalSize else bufferSize
-    }
-
     /**
      * 根据[artifactName]生成ZipEntry
      */

diff --git a/src/backend/common/common-generic/build.gradle.kts b/src/backend/common/common-generic/build.gradle.kts
@@ -0,0 +1,34 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2020 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+dependencies {
+    api(project(":common:common-artifact:artifact-service"))
+}
diff --git a/...figuration/AutoIndexRepositorySettings.kt → ...figuration/AutoIndexRepositorySettings.kt b/...figuration/AutoIndexRepositorySettings.kt → ...figuration/AutoIndexRepositorySettings.kt
@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.generic.artifact.configuration
+package com.tencent.bkrepo.common.generic.configuration
 
 import com.tencent.bkrepo.common.artifact.pojo.configuration.RepositoryConfiguration
 

diff --git a/...urity/src/main/kotlin/com/tencent/bkrepo/common/security/manager/AuthenticationManager.kt b/...urity/src/main/kotlin/com/tencent/bkrepo/common/security/manager/AuthenticationManager.kt
@@ -92,6 +92,13 @@ class AuthenticationManager(
     fun findUserAccount(userId: String): UserInfo? {
         return serviceUserClient.userInfoById(userId).data
     }
+    /**
+     * 根据用户id[userId]查询用户密码
+     * 当用户不存在时返回`null`
+     */
+    fun findUserPwd(userId: String): String? {
+        return serviceUserClient.userPwdById(userId).data
+    }
 
     fun findOauthToken(accessToken: String): OauthToken? {
         return serviceOauthAuthorizationClient.getToken(accessToken).data

diff --git a/src/backend/generic/biz-generic/build.gradle.kts b/src/backend/generic/biz-generic/build.gradle.kts
@@ -33,5 +33,6 @@ dependencies {
     api(project(":generic:api-generic"))
     api(project(":common:common-redis"))
     api(project(":analyst:api-analyst"))
+    api(project(":common:common-generic"))
     api(project(":common:common-artifact:artifact-service"))
 }
diff --git a/...generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DownloadService.kt b/...generic/biz-generic/src/main/kotlin/com/tencent/bkrepo/generic/service/DownloadService.kt
@@ -44,10 +44,10 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadCon
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactQueryContext
 import com.tencent.bkrepo.common.artifact.repository.core.ArtifactService
 import com.tencent.bkrepo.common.artifact.view.ViewModelService
+import com.tencent.bkrepo.common.generic.configuration.AutoIndexRepositorySettings
 import com.tencent.bkrepo.common.query.model.QueryModel
 import com.tencent.bkrepo.common.service.util.HttpContextHolder
 import com.tencent.bkrepo.generic.artifact.GenericArtifactInfo
-import com.tencent.bkrepo.generic.artifact.configuration.AutoIndexRepositorySettings
 import com.tencent.bkrepo.generic.artifact.context.GenericArtifactSearchContext
 import com.tencent.bkrepo.generic.constant.GenericMessageCode
 import com.tencent.bkrepo.repository.api.NodeClient