Merge pull request #1544 from jsonwan/github_fix/third_file

bugfix: 纯IPv6环境分发本地文件/第三方文件一直卡住 #1530
TencentBlueKing · Nov 28, 2022 · 1927a66 · 1927a66
2 parents 1b77e50 + 44ed20e
commit 1927a66
Show file tree

Hide file tree

Showing 11 changed files with 203 additions and 1 deletion.
diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties
@@ -101,6 +101,7 @@
 1241011=尚未支持的功能
 1241012=请求参数不合法，原因：{0}
 1241013=该功能在业务集下暂不支持
+1241014=IPv6地址段不合法，用于校验的正则表达式：^[0-9a-fA-F]{1,4}$
 
 ##业务错误-作业管理(job-manage)
 1243001=脚本不存在

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties
@@ -100,6 +100,7 @@
 1241011=Not support yet
 1241012=Invalid request parameter, reason: {0}
 1241013=Not support for biz set yet
+1241014=IPv6 seq is invalid, valid regex: ^[0-9a-fA-F]{1,4}$
 
 ## Business error - job-manage
 1243001=Script not exist

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties
@@ -100,6 +100,7 @@
 1241011=Not support yet
 1241012=Invalid request parameter, reason: {0}
 1241013=Not support for biz set yet
+1241014=IPv6 seq is invalid, valid regex: ^[0-9a-fA-F]{1,4}$
 
 ## Business error - job-manage
 1243001=Script not exist

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties
@@ -101,6 +101,7 @@
 1241011=尚未支持的功能
 1241012=请求参数不合法，原因：{0}
 1241013=该功能在业务集下暂不支持
+1241014=IPv6地址段不合法，用于校验的正则表达式：^[0-9a-fA-F]{1,4}$
 
 ##业务错误-作业管理(job-manage)
 1243001=脚本不存在

diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties
@@ -101,6 +101,7 @@
 1241011=尚未支持的功能
 1241012=请求参数不合法，原因：{0}
 1241013=该功能在业务集下暂不支持
+1241014=IPv6地址段不合法，用于校验的正则表达式：^[0-9a-fA-F]{1,4}$
 
 ##业务错误-作业管理(job-manage)
 1243001=脚本不存在

diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java
@@ -79,6 +79,8 @@ public class ErrorCode {
     public static final int ILLEGAL_PARAM_WITH_REASON = 1241012;
     // 该功能暂不支持业务集
     public static final int NOT_SUPPORT_FEATURE_FOR_BIZ_SET = 1241013;
+    // IPv6地址段不合法，用于校验的正则表达式：[0-9a-fA-F]{1,4}
+    public static final int INVALID_IPV6_SEQ = 1241014;
     // 业务通用 end
 
     // 配置服务 start

diff --git a/...ons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidIpv6SeqException.java b/...ons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidIpv6SeqException.java
@@ -0,0 +1,70 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.common.exception;
+
+import com.tencent.bk.job.common.model.error.ErrorType;
+import lombok.Getter;
+import lombok.ToString;
+
+/**
+ * IPv6地址段不合法
+ */
+@Getter
+@ToString
+public class InvalidIpv6SeqException extends ServiceException {
+
+    public InvalidIpv6SeqException(Integer errorCode) {
+        super(ErrorType.INTERNAL, errorCode);
+    }
+
+    public InvalidIpv6SeqException(Integer errorCode, Object[] errorParams) {
+        super(ErrorType.INTERNAL, errorCode, errorParams);
+    }
+
+    public InvalidIpv6SeqException(String message, Integer errorCode) {
+        super(message, ErrorType.INTERNAL, errorCode);
+    }
+
+    public InvalidIpv6SeqException(String message, Integer errorCode, Object[] errorParams) {
+        super(message, ErrorType.INTERNAL, errorCode, errorParams);
+    }
+
+    public InvalidIpv6SeqException(Throwable cause, Integer errorCode) {
+        super(cause, ErrorType.INTERNAL, errorCode);
+    }
+
+    public InvalidIpv6SeqException(Throwable cause, Integer errorCode, Object[] errorParams) {
+        super(cause, ErrorType.INTERNAL, errorCode, errorParams);
+    }
+
+    public InvalidIpv6SeqException(String message, Throwable cause, Integer errorCode) {
+        super(message, cause, ErrorType.INTERNAL, errorCode);
+    }
+
+    public InvalidIpv6SeqException(String message, Throwable cause, Integer errorCode,
+                                   Object[] errorParams) {
+        super(message, cause, ErrorType.INTERNAL, errorCode, errorParams);
+    }
+}
diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java
@@ -24,6 +24,8 @@
 
 package com.tencent.bk.job.common.util.ip;
 
+import com.tencent.bk.job.common.constant.ErrorCode;
+import com.tencent.bk.job.common.exception.InvalidIpv6SeqException;
 import com.tencent.bk.job.common.model.dto.HostDTO;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -205,6 +207,12 @@ public static boolean compileIP(String ip) {
         return matcher.matches();
     }
 
+    /**
+     * 从网卡获取首个机器IP，优先获取IPv4地址
+     * 若获取到的值为v6协议的IP，默认提供完整无压缩的IPv6地址
+     *
+     * @return 首个机器IP地址
+     */
     public static String getFirstMachineIP() {
         return getFirstMachineIpPreferV4();
     }
@@ -216,7 +224,8 @@ private static String getFirstMachineIpPreferV4() {
         }
         Map<String, String> ipv6Map = getMachineIPv6Map();
         if (!ipv6Map.isEmpty()) {
-            return ipv6Map.values().iterator().next();
+            String ipv6 = ipv6Map.values().iterator().next();
+            return getFullIpv6ByCompressedOne(ipv6);
         }
         log.error("no available ip, plz check net interface");
         return null;
@@ -361,4 +370,67 @@ public static String inferProtocolByIp(String ip) {
         }
         return PROTOCOL_IP_V4;
     }
+
+    /**
+     * 有压缩的IPv6地址转换成完整无压缩的IPv6
+     *
+     * @param compressedIpv6 有压缩的IPv6地址
+     * @return 完整无压缩的IPv6地址
+     */
+    public static String getFullIpv6ByCompressedOne(String compressedIpv6) {
+        if (StringUtils.isEmpty(compressedIpv6)) {
+            return compressedIpv6;
+        }
+        String[] finalSeqArr = new String[]{"0000", "0000", "0000", "0000", "0000", "0000", "0000", "0000"};
+        // 连续0标识符
+        String continueZeroToken = "::";
+        // 段之间分隔符
+        String seqSeparator = ":";
+        // 一个IPv6地址最多有8段
+        int maxSeqNum = 8;
+        // ::开头的IP首位补0便于后续统一分割处理
+        if (compressedIpv6.startsWith(continueZeroToken)) {
+            compressedIpv6 = "0" + compressedIpv6;
+        }
+        // ::结尾的IP末位补0便于后续统一分割处理
+        if (compressedIpv6.endsWith(continueZeroToken)) {
+            compressedIpv6 = compressedIpv6 + "0";
+        }
+        // 统一分割、解析
+        if (compressedIpv6.contains(continueZeroToken)) {
+            String[] seqArr = compressedIpv6.split(continueZeroToken);
+            String[] leftSeqArr = seqArr[0].split(seqSeparator);
+            for (int i = 0; i < leftSeqArr.length && i < maxSeqNum; i++) {
+                finalSeqArr[i] = getStandardIpv6Seq(leftSeqArr[i]);
+            }
+            String[] rightSeqArr = seqArr[1].split(seqSeparator);
+            for (int i = 0; i < rightSeqArr.length && i < maxSeqNum; i++) {
+                finalSeqArr[i + maxSeqNum - rightSeqArr.length] = getStandardIpv6Seq(rightSeqArr[i]);
+            }
+        } else {
+            String[] seqArr = compressedIpv6.split(seqSeparator);
+            for (int i = 0; i < seqArr.length && i < maxSeqNum; i++) {
+                finalSeqArr[i] = getStandardIpv6Seq(seqArr[i]);
+            }
+        }
+        return StringUtils.join(finalSeqArr, ":");
+    }
+
+    private static final Pattern ipv6SeqPattern = Pattern.compile("^[A-Fa-f0-9]{1,4}$");
+
+    /**
+     * 获取含有4个字符的Ipv6标准段，不足4字符则添加前缀0
+     *
+     * @param ipv6Seq ipv6地址的一段
+     * @return 4个字符的标准段
+     */
+    public static String getStandardIpv6Seq(String ipv6Seq) {
+        String template = "0000";
+        Matcher m = ipv6SeqPattern.matcher(ipv6Seq);
+        if (!m.matches()) {
+            log.warn("invalid ipv6Seq:{}, valid pattern:{}", ipv6Seq, ipv6SeqPattern.pattern());
+            throw new InvalidIpv6SeqException(ErrorCode.INVALID_IPV6_SEQ);
+        }
+        return template.substring(0, template.length() - ipv6Seq.length()) + ipv6Seq;
+    }
 }
diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/IpUtilsTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/IpUtilsTest.java
@@ -28,6 +28,7 @@
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
 
 public class IpUtilsTest {
     @Test
@@ -63,8 +64,10 @@ void testCheckIpv6() {
         assertThat(IpUtils.checkIpv6("01:23:45:67:89:ab:cd:ef")).isTrue();
         assertThat(IpUtils.checkIpv6("001:023:45:67:89:ab:cd:ef")).isTrue();
         assertThat(IpUtils.checkIpv6("1101:23:45:67:89:ab:cd:ef")).isTrue();
+        assertThat(IpUtils.checkIpv6("::192.168.0.1")).isTrue();
         assertThat(IpUtils.checkIpv6("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue();
         assertThat(IpUtils.checkIpv6("0ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isFalse();
+        assertThat(IpUtils.checkIpv6("::192.168.0.1/96")).isFalse();
         assertThat(IpUtils.checkIpv6("ff:ff:ff:ff:ff:ff:ff:fg")).isFalse();
         assertThat(IpUtils.checkIpv6("1::1::1")).isFalse();
         assertThat(IpUtils.checkIpv6("1::1::123")).isFalse();
@@ -116,4 +119,45 @@ void testInferProtocolByIp() {
         assertThat(IpUtils.inferProtocolByIp("::1")).isNotEqualTo(IpUtils.PROTOCOL_IP_V4);
         assertThat(IpUtils.inferProtocolByIp("01:23:45:67:89:ab:cd:ef")).isNotEqualTo(IpUtils.PROTOCOL_IP_V4);
     }
+
+    @Test
+    void testGetStandardIpv6Seq() {
+        assertThat(IpUtils.getStandardIpv6Seq("0")).isEqualTo("0000");
+        assertThat(IpUtils.getStandardIpv6Seq("1")).isEqualTo("0001");
+        assertThat(IpUtils.getStandardIpv6Seq("1001")).isEqualTo("1001");
+        assertThat(IpUtils.getStandardIpv6Seq("ff")).isEqualTo("00ff");
+        assertThatThrownBy(() -> IpUtils.getStandardIpv6Seq(".*gg"));
+        assertThatThrownBy(() -> IpUtils.getStandardIpv6Seq("aagg"));
+        assertThatThrownBy(() -> IpUtils.getStandardIpv6Seq("aabbcc"));
+    }
+
+    @Test
+    void testGetFullIpv6ByCompressedOne() {
+        assertThat(IpUtils.getFullIpv6ByCompressedOne(null)).isEqualTo(null);
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("")).isEqualTo("");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("::"))
+            .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0000");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("::1"))
+            .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("0::1"))
+            .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("00::1"))
+            .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("0000::1"))
+            .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001");
+        assertThatThrownBy(() -> IpUtils.getFullIpv6ByCompressedOne("00000::1"));
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("1:0000::1"))
+            .isEqualTo("0001:0000:0000:0000:0000:0000:0000:0001");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("1:1:1:1:1:1:1:1"))
+            .isEqualTo("0001:0001:0001:0001:0001:0001:0001:0001");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("1:1:1:0:0:1:1:1"))
+            .isEqualTo("0001:0001:0001:0000:0000:0001:0001:0001");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("ffff:1:1:0:0:1:1:ffff"))
+            .isEqualTo("ffff:0001:0001:0000:0000:0001:0001:ffff");
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("FFFF:1:1:0:0:1:1:FFFF"))
+            .isEqualTo("FFFF:0001:0001:0000:0000:0001:0001:FFFF");
+        assertThatThrownBy(() -> IpUtils.getFullIpv6ByCompressedOne("aaffff:1:1:0:0:1:1:bbffff"));
+        assertThat(IpUtils.getFullIpv6ByCompressedOne("ffff:0001:0001:0000:0000:0001:0001:ffff"))
+            .isEqualTo("ffff:0001:0001:0000:0000:0001:0001:ffff");
+    }
 }
diff --git a/...e-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java b/...e-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java
@@ -29,6 +29,7 @@
 import com.google.common.cache.LoadingCache;
 import com.tencent.bk.job.common.gse.service.AgentStateClient;
 import com.tencent.bk.job.common.model.dto.HostDTO;
+import com.tencent.bk.job.common.util.ip.IpUtils;
 import com.tencent.bk.job.execute.engine.consts.Consts;
 import com.tencent.bk.job.execute.model.ServersDTO;
 import com.tencent.bk.job.execute.service.AgentService;
@@ -118,8 +119,12 @@ private HostDTO getAgentBindHost() {
             }
             ServiceHostDTO host;
             if (physicalMachineMultiIp.contains(":")) {
+                // IPv6地址
+                // 首先转为完整无压缩格式
+                physicalMachineMultiIp = IpUtils.getFullIpv6ByCompressedOne(physicalMachineMultiIp);
                 host = getServiceHostByMultiIpv6(physicalMachineMultiIp);
             } else {
+                // IPv4地址
                 host = getServiceHostByMultiIpv4(physicalMachineMultiIp);
             }
             if (host == null) {

diff --git a/...rker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/EnvironmentService.java b/...rker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/EnvironmentService.java
@@ -98,6 +98,10 @@ public Pair<String, String> getInnerProtocolAndIp() {
         } else {
             innerIpProtocol = protocolInferredByIp;
         }
+        // 将IPv6地址转为完整无压缩格式
+        if (IpUtils.PROTOCOL_IP_V6.equalsIgnoreCase(innerIpProtocol)) {
+            innerIp = IpUtils.getFullIpv6ByCompressedOne(innerIp);
+        }
         return Pair.of(innerIpProtocol, innerIp);
     }
 }