From c07eb5af3ec8b2159e524fed4dc83b04673a1436 Mon Sep 17 00:00:00 2001 From: winghv Date: Wed, 20 Sep 2023 21:32:43 +0800 Subject: [PATCH 1/3] [fix][chore] Start parameter compatible JDK version(8-17) --- bin/datavines-daemon.sh | 2 +- bin/datavines-submit.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/datavines-daemon.sh b/bin/datavines-daemon.sh index 146e539c8..6c492c788 100644 --- a/bin/datavines-daemon.sh +++ b/bin/datavines-daemon.sh @@ -56,7 +56,7 @@ export DATAVINES_LOG_DIR=$DATAVINES_HOME/logs export DATAVINES_CONF_DIR=$DATAVINES_HOME/conf export DATAVINES_LIB_JARS=$DATAVINES_HOME/libs/* -export DATAVINES_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DATAVINES_OPTS="-server -Xmx16g -Xms1g -XX:+UseG1GC -XX:G1HeapRegionSize=8M" export STOP_TIMEOUT=5 if [ ! -d "$DATAVINES_LOG_DIR" ]; then diff --git a/bin/datavines-submit.sh b/bin/datavines-submit.sh index 0634d47d7..57020a1af 100644 --- a/bin/datavines-submit.sh +++ b/bin/datavines-submit.sh @@ -36,7 +36,7 @@ source /etc/profile export JAVA_HOME=$JAVA_HOME export DATAVINES_LIB_JARS=$DATAVINES_HOME/libs/ -export DATAVINES_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DATAVINES_OPTS="-server -Xmx16g -Xms1g -XX:+UseG1GC -XX:G1HeapRegionSize=8M" cd $DATAVINES_HOME From f6fb92e0db98915011dc64feacce521a21d3e4ac Mon Sep 17 00:00:00 2001 From: winghv Date: Tue, 26 Sep 2023 20:39:51 +0800 Subject: [PATCH 2/3] [Improvement][Common] Obtain resource information in the k8s environment. mysql Index limits on safe mode --- .../io/datavines/common/utils/OSUtils.java | 23 +++++++++++++++---- scripts/sql/datavines-mysql.sql | 5 ++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/datavines-common/src/main/java/io/datavines/common/utils/OSUtils.java b/datavines-common/src/main/java/io/datavines/common/utils/OSUtils.java index 87932995c..2fc702e1a 100644 --- a/datavines-common/src/main/java/io/datavines/common/utils/OSUtils.java +++ b/datavines-common/src/main/java/io/datavines/common/utils/OSUtils.java @@ -16,12 +16,14 @@ */ package io.datavines.common.utils; -import io.datavines.common.utils.StringUtils; +import io.datavines.common.CommonConstants; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; import java.math.RoundingMode; import java.net.InetAddress; import java.net.UnknownHostException; @@ -79,8 +81,15 @@ public static double memoryUsage() { * @return available Physical Memory Size, unit: G */ public static double availablePhysicalMemorySize() { - GlobalMemory memory = hal.getMemory(); - double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) /1024.0/1024/1024; + double availablePhysicalMemorySize; + + if (CommonConstants.KUBERNETES_MODE) { + long freeMemory = Runtime.getRuntime().freeMemory(); + availablePhysicalMemorySize = freeMemory /1024.0/1024/1024; + } else { + GlobalMemory memory = hal.getMemory(); + availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) /1024.0/1024/1024; + } DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); @@ -110,7 +119,13 @@ public static double totalMemorySize() { * @return load average */ public static double loadAverage() { - double loadAverage = hal.getProcessor().getSystemLoadAverage(); + double loadAverage; + if (CommonConstants.KUBERNETES_MODE) { + OperatingSystemMXBean operatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean(); + loadAverage = operatingSystemMXBean.getSystemLoadAverage(); + } else { + loadAverage = hal.getProcessor().getSystemLoadAverage(); + } DecimalFormat df = new DecimalFormat(TWO_DECIMAL); diff --git a/scripts/sql/datavines-mysql.sql b/scripts/sql/datavines-mysql.sql index c433a91b8..46b0e8c2d 100644 --- a/scripts/sql/datavines-mysql.sql +++ b/scripts/sql/datavines-mysql.sql @@ -643,8 +643,9 @@ CREATE TABLE `dv_registry_lock` `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time', PRIMARY KEY (`id`), - unique (`lock_key`) -) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; + UNIQUE KEY (`uniq_lock_key`) (`lock_key`) USING BTREE, + KEY `idx_upt` (`update_time`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT='注册锁'; -- ---------------------------- -- Table structure for dv_sla From de4f07e78b941f3632815c2d9159a9148cdec492 Mon Sep 17 00:00:00 2001 From: winghv Date: Wed, 27 Sep 2023 09:50:32 +0800 Subject: [PATCH 3/3] [fix][scripts] mysql scripts grammatical errors --- scripts/sql/datavines-mysql.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/sql/datavines-mysql.sql b/scripts/sql/datavines-mysql.sql index 46b0e8c2d..663269451 100644 --- a/scripts/sql/datavines-mysql.sql +++ b/scripts/sql/datavines-mysql.sql @@ -643,7 +643,7 @@ CREATE TABLE `dv_registry_lock` `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time', PRIMARY KEY (`id`), - UNIQUE KEY (`uniq_lock_key`) (`lock_key`) USING BTREE, + UNIQUE KEY `uniq_lock_key` (`lock_key`) USING BTREE, KEY `idx_upt` (`update_time`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT='注册锁'; @@ -794,7 +794,7 @@ CREATE TABLE `dv_config` ( `update_by` bigint(20) NOT NULL COMMENT '更新用户ID', `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=utf8mb4 COMMENT='配置'; +) ENGINE=InnoDB AUTO_INCREMENT=23 DEFAULT CHARSET=utf8mb4 COMMENT='配置'; INSERT INTO `dv_config` VALUES ('1', '-1', 'data.quality.jar.name', '/libs/datavines-engine-spark-core-1.0.0-SNAPSHOT.jar', '1', '1', '2023-09-02 16:52:56', '1', '2023-09-03 09:56:12'); INSERT INTO `dv_config` VALUES ('2', '-1', 'yarn.mode', 'standalone', '1', '1', '2023-09-02 18:28:59', '1', '2023-09-03 12:46:24');