From 9d2fe7316ab4eb5b584772613908177cc7c0688f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=88=91=E7=9A=84=E5=A4=B4=E5=83=8F=E6=98=AF=E4=B8=8A?= =?UTF-8?q?=E5=B8=9D?= <35247309+329724814@users.noreply.github.com> Date: Mon, 17 Apr 2023 20:21:02 +0800 Subject: [PATCH] replace ArcticMetaStore.java's FastJson to Jackson --- .../arctic/ams/server/ArcticMetaStore.java | 135 +++++++++--------- .../arctic/ams/server/utils/JacksonUtils.java | 133 +++++++++++++++++ .../arctic/ams/server/utils/YamlUtils.java | 8 +- ams/ams-server/src/test/java/JacksonTest.java | 121 ++++++++++++++++ ams/ams-server/src/test/resources/test.yaml | 18 +-- 5 files changed, 336 insertions(+), 79 deletions(-) create mode 100644 ams/ams-server/src/main/java/com/netease/arctic/ams/server/utils/JacksonUtils.java create mode 100644 ams/ams-server/src/test/java/JacksonTest.java diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/ArcticMetaStore.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/ArcticMetaStore.java index f467c65233..9fd2025bfb 100644 --- a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/ArcticMetaStore.java +++ b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/ArcticMetaStore.java @@ -18,8 +18,8 @@ package com.netease.arctic.ams.server; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; import com.netease.arctic.ams.api.ArcticTableMetastore; import com.netease.arctic.ams.api.CatalogMeta; @@ -45,11 +45,7 @@ import com.netease.arctic.ams.server.service.impl.FileInfoCacheService; import com.netease.arctic.ams.server.service.impl.OptimizeExecuteService; import com.netease.arctic.ams.server.service.impl.RuntimeDataExpireService; -import com.netease.arctic.ams.server.utils.AmsUtils; -import com.netease.arctic.ams.server.utils.SecurityUtils; -import com.netease.arctic.ams.server.utils.ThreadPool; -import com.netease.arctic.ams.server.utils.UpdateTool; -import com.netease.arctic.ams.server.utils.YamlUtils; +import com.netease.arctic.ams.server.utils.*; import com.netease.arctic.utils.ConfigurationFileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.recipes.leader.LeaderLatchListener; @@ -81,7 +77,7 @@ public class ArcticMetaStore { public static final Logger LOG = LoggerFactory.getLogger(ArcticMetaStore.class); public static Configuration conf; - private static JSONObject yamlConfig; + private static JsonNode yamlConfig; private static TServer server; private static final List residentThreads = new ArrayList<>(); private static HighAvailabilityServices haService = null; @@ -98,7 +94,7 @@ public static void main(String[] args) throws Throwable { } public static LinkedHashMap getSystemSettingFromYaml() { - JSONObject systemConfig = yamlConfig.getJSONObject(ConfigFileProperties.SYSTEM_CONFIG); + ObjectNode systemConfig = (ObjectNode) yamlConfig.get(ConfigFileProperties.SYSTEM_CONFIG); LinkedHashMap config = new LinkedHashMap(); systemConfig.put(ArcticMetaStoreConf.ARCTIC_HOME.key(), getArcticHome()); @@ -106,47 +102,49 @@ public static LinkedHashMap getSystemSettingFromYaml() { if (systemThriftPort == null) { systemConfig.put( ArcticMetaStoreConf.THRIFT_BIND_PORT.key(), - systemConfig.getInteger(ArcticMetaStoreConf.THRIFT_BIND_PORT.key())); + JacksonUtils.getInteger(systemConfig, ArcticMetaStoreConf.THRIFT_BIND_PORT.key())); } else { systemConfig.put(ArcticMetaStoreConf.THRIFT_BIND_PORT.key(), Integer.parseInt(systemThriftPort)); } validateConfig(systemConfig); - config.putAll(systemConfig); + config.putAll(JacksonUtils.parseObject(JacksonUtils.toJSONString(systemConfig), Map.class)); //extension properties + String extensionProStr = JacksonUtils.getString(yamlConfig, ConfigFileProperties.SYSTEM_EXTENSION_CONFIG); Map extensionPro = - yamlConfig.getObject(ConfigFileProperties.SYSTEM_EXTENSION_CONFIG, Map.class) == null ? new HashMap<>() : - yamlConfig.getObject(ConfigFileProperties.SYSTEM_EXTENSION_CONFIG, Map.class); + JacksonUtils.parseObject(extensionProStr, Map.class) == null ? new HashMap<>() : + JacksonUtils.parseObject(extensionProStr, Map.class); config.put(ArcticMetaStoreConf.SYSTEM_EXTENSION_PROPERTIES.key(), extensionPro); return config; } - public static void validateConfig(JSONObject systemConfig) { - if (!systemConfig.containsKey(ArcticMetaStoreConf.THRIFT_BIND_HOST_PREFIX.key())) { + public static void validateConfig(ObjectNode systemConfig) { + if (!systemConfig.has(ArcticMetaStoreConf.THRIFT_BIND_HOST_PREFIX.key())) { throw new RuntimeException("configuration " + ArcticMetaStoreConf.THRIFT_BIND_HOST_PREFIX.key() + " must be set"); } InetAddress inetAddress = AmsUtils.getLocalHostExactAddress( - systemConfig.getString(ArcticMetaStoreConf.THRIFT_BIND_HOST_PREFIX.key())); + JacksonUtils.getString(systemConfig, ArcticMetaStoreConf.THRIFT_BIND_HOST_PREFIX.key())); if (inetAddress == null) { throw new RuntimeException("can't find host address start with " + - systemConfig.getString(ArcticMetaStoreConf.THRIFT_BIND_HOST_PREFIX.key())); + JacksonUtils.getString(systemConfig, ArcticMetaStoreConf.THRIFT_BIND_HOST_PREFIX.key())); } systemConfig.put(ArcticMetaStoreConf.THRIFT_BIND_HOST.key(), inetAddress.getHostAddress()); //mysql config - if (systemConfig.getString(ArcticMetaStoreConf.DB_TYPE.key()).equalsIgnoreCase("mysql")) { - if (!systemConfig.containsKey(ArcticMetaStoreConf.MYBATIS_CONNECTION_PASSWORD.key()) || - !systemConfig.containsKey(ArcticMetaStoreConf.MYBATIS_CONNECTION_USER_NAME.key())) { + String dbType = JacksonUtils.getString(systemConfig, ArcticMetaStoreConf.DB_TYPE.key()); + if (("mysql").equalsIgnoreCase(dbType)) { + if (!systemConfig.has(ArcticMetaStoreConf.MYBATIS_CONNECTION_PASSWORD.key()) || + !systemConfig.has(ArcticMetaStoreConf.MYBATIS_CONNECTION_USER_NAME.key())) { throw new RuntimeException("username and password must be configured if the database type is mysql"); } } //HA config - if (systemConfig.containsKey(ArcticMetaStoreConf.HA_ENABLE.key()) && - systemConfig.getBoolean(ArcticMetaStoreConf.HA_ENABLE.key())) { - if (!systemConfig.containsKey(ArcticMetaStoreConf.ZOOKEEPER_SERVER.key())) { + if (systemConfig.has(ArcticMetaStoreConf.HA_ENABLE.key()) && + JacksonUtils.getBoolean(systemConfig, ArcticMetaStoreConf.HA_ENABLE.key())) { + if (!systemConfig.has(ArcticMetaStoreConf.ZOOKEEPER_SERVER.key())) { throw new RuntimeException(ArcticMetaStoreConf.ZOOKEEPER_SERVER.key() + " must be configured when you enable " + "the ams high availability"); } @@ -483,7 +481,7 @@ private static void checkLeader() { !haService.getMaster().equals(haService.getNodeInfo( conf.getString(ArcticMetaStoreConf.THRIFT_BIND_HOST), conf.getInteger(ArcticMetaStoreConf.THRIFT_BIND_PORT)))) { - LOG.info("there is not leader, the leader is " + JSONObject.toJSONString(haService.getMaster())); + LOG.info("there is not leader, the leader is " + JacksonUtils.toJSONString(haService.getMaster())); failover(); } } catch (Exception e) { @@ -503,77 +501,78 @@ private static Configuration initSystemConfig() { } private static void initCatalogConfig() throws IOException { - JSONArray catalogs = yamlConfig.getJSONArray(ConfigFileProperties.CATALOG_LIST); + JsonNode catalogs = yamlConfig.findValue(ConfigFileProperties.CATALOG_LIST).get(0); List catalogMetas = Lists.newArrayList(); for (int i = 0; i < catalogs.size(); i++) { CatalogMeta catalogMeta = new CatalogMeta(); - JSONObject catalog = catalogs.getJSONObject(i); - catalogMeta.catalogName = catalog.getString(ConfigFileProperties.CATALOG_NAME); - catalogMeta.catalogType = catalog.getString(ConfigFileProperties.CATALOG_TYPE); + JsonNode catalog = catalogs.get(i); + catalogMeta.catalogName = JacksonUtils.getString(catalog, ConfigFileProperties.CATALOG_NAME); + catalogMeta.catalogType = JacksonUtils.getString(catalog, ConfigFileProperties.CATALOG_TYPE); - if (catalog.containsKey(ConfigFileProperties.CATALOG_STORAGE_CONFIG)) { + if (catalog.has(ConfigFileProperties.CATALOG_STORAGE_CONFIG)) { Map storageConfig = new HashMap<>(); - JSONObject catalogStorageConfig = catalog.getJSONObject(ConfigFileProperties.CATALOG_STORAGE_CONFIG); - if (catalogStorageConfig.containsKey(ConfigFileProperties.CATALOG_STORAGE_TYPE)) { + JsonNode catalogStorageConfig = catalog.get(ConfigFileProperties.CATALOG_STORAGE_CONFIG); + if (catalogStorageConfig.has(ConfigFileProperties.CATALOG_STORAGE_TYPE)) { storageConfig.put( CatalogMetaProperties.STORAGE_CONFIGS_KEY_TYPE, - catalogStorageConfig.getString(ConfigFileProperties.CATALOG_STORAGE_TYPE)); + JacksonUtils.getString(catalogStorageConfig, ConfigFileProperties.CATALOG_STORAGE_TYPE)); } storageConfig.put( CatalogMetaProperties.STORAGE_CONFIGS_KEY_CORE_SITE, ConfigurationFileUtils.encodeXmlConfigurationFileWithBase64( - catalogStorageConfig.getString(ConfigFileProperties.CATALOG_CORE_SITE))); + JacksonUtils.getString(catalogStorageConfig, ConfigFileProperties.CATALOG_CORE_SITE))); storageConfig.put( CatalogMetaProperties.STORAGE_CONFIGS_KEY_HDFS_SITE, ConfigurationFileUtils.encodeXmlConfigurationFileWithBase64( - catalogStorageConfig.getString(ConfigFileProperties.CATALOG_HDFS_SITE))); + JacksonUtils.getString(catalogStorageConfig, ConfigFileProperties.CATALOG_HDFS_SITE))); storageConfig.put( CatalogMetaProperties.STORAGE_CONFIGS_KEY_HIVE_SITE, ConfigurationFileUtils.encodeXmlConfigurationFileWithBase64( - catalogStorageConfig.getString(ConfigFileProperties.CATALOG_HIVE_SITE))); + JacksonUtils.getString(catalogStorageConfig, ConfigFileProperties.CATALOG_HIVE_SITE))); catalogMeta.storageConfigs = storageConfig; } - if (catalog.containsKey(ConfigFileProperties.CATALOG_AUTH_CONFIG)) { + if (catalog.has(ConfigFileProperties.CATALOG_AUTH_CONFIG)) { Map authConfig = new HashMap<>(); - JSONObject catalogAuthConfig = catalog.getJSONObject(ConfigFileProperties.CATALOG_AUTH_CONFIG); - if (catalogAuthConfig.containsKey(ConfigFileProperties.CATALOG_AUTH_TYPE)) { + JsonNode catalogAuthConfig = catalog.get(ConfigFileProperties.CATALOG_AUTH_CONFIG); + if (catalogAuthConfig.has(ConfigFileProperties.CATALOG_AUTH_TYPE)) { authConfig.put( CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE, - catalogAuthConfig.getString(ConfigFileProperties.CATALOG_AUTH_TYPE)); + JacksonUtils.getString(catalogAuthConfig, ConfigFileProperties.CATALOG_AUTH_TYPE)); } - if (catalogAuthConfig.getString(ConfigFileProperties.CATALOG_AUTH_TYPE) + if (JacksonUtils.getString(catalogAuthConfig, ConfigFileProperties.CATALOG_AUTH_TYPE) .equalsIgnoreCase(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE)) { - if (catalogAuthConfig.containsKey(ConfigFileProperties.CATALOG_SIMPLE_HADOOP_USERNAME)) { + if (catalogAuthConfig.has(ConfigFileProperties.CATALOG_SIMPLE_HADOOP_USERNAME)) { authConfig.put( CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME, - catalogAuthConfig.getString(ConfigFileProperties.CATALOG_SIMPLE_HADOOP_USERNAME)); + JacksonUtils.getString(catalogAuthConfig, ConfigFileProperties.CATALOG_SIMPLE_HADOOP_USERNAME)); } - } else if (catalogAuthConfig.getString(ConfigFileProperties.CATALOG_AUTH_TYPE) + } else if (JacksonUtils.getString(catalogAuthConfig, ConfigFileProperties.CATALOG_AUTH_TYPE) .equalsIgnoreCase(CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_KERBEROS)) { - if (catalogAuthConfig.containsKey(ConfigFileProperties.CATALOG_KEYTAB)) { + if (catalogAuthConfig.has(ConfigFileProperties.CATALOG_KEYTAB)) { authConfig.put( CatalogMetaProperties.AUTH_CONFIGS_KEY_KEYTAB, ConfigurationFileUtils.encodeConfigurationFileWithBase64( - catalogAuthConfig.getString(ConfigFileProperties.CATALOG_KEYTAB))); + JacksonUtils.getString(catalogAuthConfig, ConfigFileProperties.CATALOG_KEYTAB))); } - if (catalogAuthConfig.containsKey(ConfigFileProperties.CATALOG_KRB5)) { + if (catalogAuthConfig.has(ConfigFileProperties.CATALOG_KRB5)) { authConfig.put( CatalogMetaProperties.AUTH_CONFIGS_KEY_KRB5, ConfigurationFileUtils.encodeConfigurationFileWithBase64( - catalogAuthConfig.getString(ConfigFileProperties.CATALOG_KRB5))); + JacksonUtils.getString(catalogAuthConfig, ConfigFileProperties.CATALOG_KRB5))); } - if (catalogAuthConfig.containsKey(ConfigFileProperties.CATALOG_PRINCIPAL)) { + if (catalogAuthConfig.has(ConfigFileProperties.CATALOG_PRINCIPAL)) { authConfig.put( CatalogMetaProperties.AUTH_CONFIGS_KEY_PRINCIPAL, - catalogAuthConfig.getString(ConfigFileProperties.CATALOG_PRINCIPAL)); + JacksonUtils.getString(catalogAuthConfig, ConfigFileProperties.CATALOG_PRINCIPAL)); } } catalogMeta.authConfigs = authConfig; } - if (catalog.containsKey(ConfigFileProperties.CATALOG_PROPERTIES)) { - catalogMeta.catalogProperties = catalog.getObject(ConfigFileProperties.CATALOG_PROPERTIES, Map.class); + if (catalog.has(ConfigFileProperties.CATALOG_PROPERTIES)) { + catalogMeta.catalogProperties = JacksonUtils.parseObject(JacksonUtils.getString(catalog, + ConfigFileProperties.CATALOG_PROPERTIES), Map.class); } catalogMetas.add(catalogMeta); } @@ -591,14 +590,15 @@ private static void initConfig() { } private static void initContainerConfig() { - JSONArray containers = yamlConfig.getJSONArray(ConfigFileProperties.CONTAINER_LIST); + JsonNode containers = yamlConfig.findValues(ConfigFileProperties.CONTAINER_LIST).get(0); for (int i = 0; i < containers.size(); i++) { - JSONObject optimize = containers.getJSONObject(i); + JsonNode optimize = containers.get(i); Container container = new Container(); - container.setName(optimize.getString(ConfigFileProperties.CONTAINER_NAME)); - container.setType(optimize.getString(ConfigFileProperties.CONTAINER_TYPE)); - if (optimize.containsKey(ConfigFileProperties.CONTAINER_PROPERTIES)) { - container.setProperties(optimize.getObject(ConfigFileProperties.CONTAINER_PROPERTIES, Map.class)); + container.setName(JacksonUtils.getString(optimize, ConfigFileProperties.CONTAINER_NAME)); + container.setType(JacksonUtils.getString(optimize, ConfigFileProperties.CONTAINER_TYPE)); + if (optimize.has(ConfigFileProperties.CONTAINER_PROPERTIES)) { + container.setProperties(JacksonUtils.parseObject(JacksonUtils.getString(optimize, + ConfigFileProperties.CONTAINER_PROPERTIES), Map.class)); } ServiceContainer.getOptimizeQueueService().insertContainer(container); @@ -606,17 +606,17 @@ private static void initContainerConfig() { } private static void initOptimizeGroupConfig() throws MetaException, NoSuchObjectException, InvalidObjectException { - JSONArray optimizeGroups = yamlConfig.getJSONArray(ConfigFileProperties.OPTIMIZE_GROUP_LIST); + JsonNode optimizeGroups = yamlConfig.findValues(ConfigFileProperties.OPTIMIZE_GROUP_LIST).get(0); List optimizeQueueMetas = ServiceContainer.getOptimizeQueueService().getQueues(); for (int i = 0; i < optimizeGroups.size(); i++) { - JSONObject optimizeGroup = optimizeGroups.getJSONObject(i); + JsonNode optimizeGroup = optimizeGroups.get(i); OptimizeQueueMeta optimizeQueueMeta = new OptimizeQueueMeta(); - optimizeQueueMeta.setName(optimizeGroup.getString(ConfigFileProperties.OPTIMIZE_GROUP_NAME)); - optimizeQueueMeta.setContainer(optimizeGroup.getString(ConfigFileProperties.OPTIMIZE_GROUP_CONTAINER)); + optimizeQueueMeta.setName(JacksonUtils.getString(optimizeGroup, ConfigFileProperties.OPTIMIZE_GROUP_NAME)); + optimizeQueueMeta.setContainer(JacksonUtils.getString(optimizeGroup, ConfigFileProperties.OPTIMIZE_GROUP_CONTAINER)); //init schedule policy String schedulePolicy = - StringUtils.trim(optimizeGroup.getString(ConfigFileProperties.OPTIMIZE_SCHEDULING_POLICY)); + StringUtils.trim(JacksonUtils.getString(optimizeGroup, ConfigFileProperties.OPTIMIZE_SCHEDULING_POLICY)); if (StringUtils.isBlank(schedulePolicy)) { schedulePolicy = ConfigFileProperties.OPTIMIZE_SCHEDULING_POLICY_QUOTA; } else if ( @@ -634,15 +634,16 @@ private static void initOptimizeGroupConfig() throws MetaException, NoSuchObject boolean checkContainer = containers.stream() .anyMatch(e -> e.getName() - .equalsIgnoreCase(optimizeGroup.getString(ConfigFileProperties.OPTIMIZE_GROUP_CONTAINER))); + .equalsIgnoreCase(JacksonUtils.getString(optimizeGroup, ConfigFileProperties.OPTIMIZE_GROUP_CONTAINER))); if (!checkContainer) { throw new NoSuchObjectException( "can not find such container config named:" + - optimizeGroup.getString(ConfigFileProperties.OPTIMIZE_GROUP_CONTAINER)); + JacksonUtils.getString(optimizeGroup, ConfigFileProperties.OPTIMIZE_GROUP_CONTAINER)); } - if (optimizeGroup.containsKey(ConfigFileProperties.OPTIMIZE_GROUP_PROPERTIES)) { + if (optimizeGroup.has(ConfigFileProperties.OPTIMIZE_GROUP_PROPERTIES)) { optimizeQueueMeta.setProperties( - optimizeGroup.getObject(ConfigFileProperties.OPTIMIZE_GROUP_PROPERTIES, Map.class)); + JacksonUtils.parseObject(JacksonUtils.getString(optimizeGroup, + ConfigFileProperties.OPTIMIZE_GROUP_PROPERTIES), Map.class)); } boolean updated = false; for (OptimizeQueueMeta meta : optimizeQueueMetas) { @@ -674,7 +675,7 @@ public void isLeader() { AmsServerInfo serverInfo = new AmsServerInfo(); serverInfo.setHost(conf.getString(ArcticMetaStoreConf.THRIFT_BIND_HOST)); serverInfo.setThriftBindPort(conf.getInteger(ArcticMetaStoreConf.THRIFT_BIND_PORT)); - zkService.setData(masterPath, JSONObject.toJSONString(serverInfo)); + zkService.setData(masterPath, JacksonUtils.toJSONString(serverInfo)); isLeader.set(true); startMetaStore(initSystemConfig()); } catch (Throwable throwable) { diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/utils/JacksonUtils.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/utils/JacksonUtils.java new file mode 100644 index 0000000000..54617242c9 --- /dev/null +++ b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/utils/JacksonUtils.java @@ -0,0 +1,133 @@ +package com.netease.arctic.ams.server.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class JacksonUtils { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final ObjectMapper INDENT_OBJECT_MAPPER = new ObjectMapper(); + + private static final String SERIALIZE_ERROR = "serialize object error"; + + private static final String DESERIALIZE_ERROR = "deserialize to object error"; + + static { + OBJECT_MAPPER.configure(SerializationFeature.WRITE_ENUMS_USING_INDEX, true); + OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + INDENT_OBJECT_MAPPER.configure(SerializationFeature.INDENT_OUTPUT, true); + } + + private JacksonUtils(){ + //do nothing + } + + public static String toIndentJSON(Object object){ + try{ + return INDENT_OBJECT_MAPPER.writeValueAsString(object); + } catch (JsonProcessingException e) { + throw new IllegalStateException(SERIALIZE_ERROR, e); + } + } + + public static String toJSONString(Object object){ + try{ + return OBJECT_MAPPER.writeValueAsString(object); + } catch (JsonProcessingException e){ + throw new IllegalStateException(SERIALIZE_ERROR, e); + } + } + + public static T parseObject(String jsonString, Class clazz){ + try { + return OBJECT_MAPPER.readValue(jsonString, clazz); + } catch (IOException e){ + throw new IllegalStateException(DESERIALIZE_ERROR, e); + } + } + + public static T parseObject(String jsonString, TypeReference typeReference){ + try { + return OBJECT_MAPPER.readValue(jsonString, typeReference); + } catch (JsonProcessingException e) { + throw new IllegalStateException(DESERIALIZE_ERROR, e); + } + } + + public static T parseObjects(String jsonString, Class clazz){ + JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametricType(ArrayList.class, clazz); + try { + return OBJECT_MAPPER.readValue(jsonString, javaType); + } catch (JsonProcessingException e) { + throw new IllegalStateException(DESERIALIZE_ERROR, e); + } + } + + public static List parseObjectsString(String jsonString){ + JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametricType(ArrayList.class, String.class); + try { + return OBJECT_MAPPER.readValue(jsonString, javaType); + } catch (JsonProcessingException e) { + throw new IllegalStateException(DESERIALIZE_ERROR, e); + } + } + + public static String getString(JsonNode jsonNode, String fieldName){ + if(jsonNode == null){ + throw new NullPointerException(); + } + if(jsonNode.get(fieldName) == null){ + return null; + } + JsonNode result = jsonNode.get(fieldName); + if(result.isTextual()){ + return jsonNode.get(fieldName).asText(); + } + return jsonNode.get(fieldName).toString(); + } + + public static Integer getInteger(JsonNode jsonNode, String fieldName) { + if(jsonNode == null){ + throw new NullPointerException(); + } + if(jsonNode.get(fieldName) == null){ + return null; + } + JsonNode value = jsonNode.get(fieldName); + if(value.isInt()){ + return value.intValue(); + } + throw new ClassCastException("can not cast to int, value : " + value); + } + + public static Boolean getBoolean(JsonNode jsonNode, String fieldName){ + if(jsonNode == null){ + throw new NullPointerException(); + } + if(jsonNode.get(fieldName) == null){ + return null; + } + JsonNode value = jsonNode.get(fieldName); + if (value.isBoolean()) { + return value.booleanValue(); + } + throw new ClassCastException("can not cast to bool, value : " + value); + } + + public static String getOrDefaultString(JsonNode jsonNode, String fieldName, String defaultValue){ + if(jsonNode == null){ + throw new NullPointerException(); + } + if(jsonNode.get(fieldName) == null){ + return defaultValue; + } + return jsonNode.get(fieldName).asText(); + } + +} diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/utils/YamlUtils.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/utils/YamlUtils.java index cb733a6bcd..04d2264b2a 100644 --- a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/utils/YamlUtils.java +++ b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/utils/YamlUtils.java @@ -18,7 +18,8 @@ package com.netease.arctic.ams.server.utils; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.yaml.snakeyaml.Yaml; import java.io.FileInputStream; @@ -29,7 +30,7 @@ public class YamlUtils { - public static JSONObject load(String path) { + public static JsonNode load(String path) { InputStream inputStream = null; try { inputStream = new FileInputStream(path); @@ -37,6 +38,7 @@ public static JSONObject load(String path) { e.printStackTrace(); } Yaml yaml = new Yaml(); - return new JSONObject(yaml.loadAs(inputStream, Map.class)); + ObjectMapper objMapper = new ObjectMapper(); + return objMapper.valueToTree(yaml.loadAs(inputStream, Map.class)); } } diff --git a/ams/ams-server/src/test/java/JacksonTest.java b/ams/ams-server/src/test/java/JacksonTest.java new file mode 100644 index 0000000000..49bef80173 --- /dev/null +++ b/ams/ams-server/src/test/java/JacksonTest.java @@ -0,0 +1,121 @@ +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.netease.arctic.ams.server.config.ConfigFileProperties; +import com.netease.arctic.ams.server.utils.JacksonUtils; +import org.junit.Test; +import org.yaml.snakeyaml.Yaml; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.*; + +public class JacksonTest { + + + + @Test + public void jacksonTest(){ + String configPath = System.getProperty("user.dir") + "/src/test/resources/test.yaml"; + JsonNode yamlConfig = load(configPath); + ObjectNode systemConfig =(ObjectNode) yamlConfig.get(ConfigFileProperties.SYSTEM_CONFIG); + systemConfig.put("test", "test1"); + LinkedHashMap config = new LinkedHashMap(); + config.putAll(JacksonUtils.parseObject(JacksonUtils.toJSONString(systemConfig), Map.class)); + JsonNode catalogs = yamlConfig.findValues(ConfigFileProperties.CONTAINER_LIST).get(0); + for(JsonNode jsonNode : catalogs){ + System.out.println(jsonNode.get("name").asText()); + } + System.out.println(systemConfig.has("arctic.ams.expire.thread.pool-size")); + System.out.println(JacksonUtils.getString(systemConfig, "aaaaa")); + System.out.println(JacksonUtils.getString(systemConfig, "arctic.ams.expire.thread.pool-size")); + System.out.println(JacksonUtils.getInteger(systemConfig, "arctic.ams.expire.thread.pool-size")); + try { + System.out.println(systemConfig.get("arctic.ams.mybatis.ConnectionDriverClassName").asText()); + }catch (Exception ex){ + ex.printStackTrace(); + } + System.out.println(JacksonUtils.getOrDefaultString(systemConfig, "c", "test")); + try{ + System.out.println(JacksonUtils.getBoolean(systemConfig,"a")); + } catch (Exception ex){ + ex.printStackTrace(); + } + + System.out.println(JacksonUtils.getBoolean(systemConfig,"arctic.ams.ha.enabled")); + System.out.println(JacksonUtils.parseObject(JacksonUtils.getString(yamlConfig, "ams"), Map.class)); + String listStr = JacksonUtils.getString(systemConfig, "d"); + System.out.println(listStr); + List e = yamlConfig.findValues("d"); + + for (Iterator it = systemConfig.fieldNames(); it.hasNext(); ) { + String key = it.next(); + if (key != null) { + String value = JacksonUtils.getString(systemConfig, key); + System.out.println(value); + } + } + + JackTest test = new JackTest(); + test.setA("good"); + test.setB("better"); + test.setC(12); + System.out.println(JacksonUtils.toJSONString(test)); + + String testJson = "{\"a\":\"good\",\"b\":\"better\",\"c\":12}"; + JackTest jack = JacksonUtils.parseObject(testJson, JackTest.class); + + + Map testDir = new HashMap<>(); + testDir.put("a","aaa"); + testDir.put("b", true); + testDir.put("c", 1); + System.out.println(JacksonUtils.toJSONString(testDir)); + } + + static class JackTest{ + + private String a; + + private String b; + + private Integer c; + + public String getA() { + return a; + } + + public void setA(String a) { + this.a = a; + } + + public String getB() { + return b; + } + + public void setB(String b) { + this.b = b; + } + + public Integer getC() { + return c; + } + + public void setC(Integer c) { + this.c = c; + } + } + + private static JsonNode load(String path) { + InputStream inputStream = null; + try { + inputStream = new FileInputStream(path); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + Yaml yaml = new Yaml(); + ObjectMapper objMapper = new ObjectMapper(); + return objMapper.valueToTree(yaml.loadAs(inputStream, Map.class)); + } +} diff --git a/ams/ams-server/src/test/resources/test.yaml b/ams/ams-server/src/test/resources/test.yaml index 1ce015c2a6..dfccfa837e 100644 --- a/ams/ams-server/src/test/resources/test.yaml +++ b/ams/ams-server/src/test/resources/test.yaml @@ -19,7 +19,7 @@ ams: #arctic.ams.database.type: mysql #HA config - #arctic.ams.ha.enabled: true + arctic.ams.ha.enabled: true #arctic.ams.cluster.name: default #arctic.ams.zookeeper.server: 127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183 @@ -41,14 +41,14 @@ containers: properties: hadoop_home: /opt/hadoop # java_home: /opt/java -# - name: flinkContainer -# type: flink -# properties: -# FLINK_HOME: /opt/flink/ #flink install home -# HADOOP_CONF_DIR: /etc/hadoop/conf/ #hadoop config dir -# HADOOP_USER_NAME: hadoop #hadoop user submit on yarn -# JVM_ARGS: -Djava.security.krb5.conf=/opt/krb5.conf #flink launch jvm args, like kerberos config when ues kerberos -# FLINK_CONF_DIR: /etc/hadoop/conf/ #flink config dir + - name: flinkContainer + type: flink + properties: + FLINK_HOME: /opt/flink/ #flink install home + HADOOP_CONF_DIR: /etc/hadoop/conf/ #hadoop config dir + HADOOP_USER_NAME: hadoop #hadoop user submit on yarn + JVM_ARGS: -Djava.security.krb5.conf=/opt/krb5.conf #flink launch jvm args, like kerberos config when ues kerberos + FLINK_CONF_DIR: /etc/hadoop/conf/ #flink config dir