Merge pull request #11 from wucheng/master

1 remove some log 2 fix classloader
tencentyun · Jul 21, 2020 · c4621b0 · c4621b0
2 parents 79294f3 + 2e48940
commit c4621b0
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 16 deletions.
diff --git a/jar/chdfs_hadoop_plugin_network-1.9.jar b/jar/chdfs_hadoop_plugin_network-1.9.jar
diff --git a/jar/chdfs_hadoop_plugin_network-2.0.jar b/jar/chdfs_hadoop_plugin_network-2.0.jar
diff --git a/jar/jar.md5 b/jar/jar.md5
@@ -1 +1 @@
-MD5 (chdfs_hadoop_plugin_network-1.9.jar) = b7ac614fb674889218de7f94589a0632
+MD5 (chdfs_hadoop_plugin_network-2.0.jar) = 53468068a299cd0b73ec5dab9c810b6d
diff --git a/pom.xml b/pom.xml
@@ -5,7 +5,7 @@
 
     <groupId>com.qcloud</groupId>
     <artifactId>chdfs_hadoop_plugin_network</artifactId>
-    <version>1.9</version>
+    <version>2.0</version>
     <packaging>jar</packaging>
 
     <name>chdfs_hadoop_plugin_network</name>
@@ -15,7 +15,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.7</maven.compiler.source>
         <maven.compiler.target>1.7</maven.compiler.target>
-        <hadoop.version>2.7.3</hadoop.version>
+        <hadoop.version>3.1.2</hadoop.version>
     </properties>
 
     <dependencies>

diff --git a/src/main/java/com/qcloud/chdfs/fs/CHDFSHadoopFileSystemAdapter.java b/src/main/java/com/qcloud/chdfs/fs/CHDFSHadoopFileSystemAdapter.java
@@ -24,7 +24,7 @@ public class CHDFSHadoopFileSystemAdapter extends FileSystem {
 
     static final String SCHEME = "ofs";
     public static final String CHDFS_DATA_TRANSFER_ENDPOINT_SUFFIX_KEY = "fs.ofs.data.transfer.endpoint.suffix";
-    private static final String MOUNT_POINT_ADDR_PATTERN = "^([a-zA-Z0-9-]+)\\.chdfs(\\.inner)?\\.([a-z0-9-]+)\\.([a-z0-9-.]+)";
+    private static final String MOUNT_POINT_ADDR_PATTERN = "^([a-zA-Z0-9-]+)\\.chdfs(-dualstack)?(\\.inner)?\\.([a-z0-9-]+)\\.([a-z0-9-.]+)";
     private static final String CHDFS_USER_APPID_KEY = "fs.ofs.user.appid";
     private static final String CHDFS_TMP_CACHE_DIR_KEY = "fs.ofs.tmp.cache.dir";
     private static final String CHDFS_META_SERVER_PORT_KEY = "fs.ofs.meta.server.port";
@@ -126,8 +126,9 @@ private boolean isJarPluginServerHttps(Configuration conf) {
 
     @Override
     public void initialize(URI name, Configuration conf) throws IOException {
+        log.debug("adapter initialize");
         this.initStartMs = System.currentTimeMillis();
-        log.info("start-init-start time: {}", initStartMs);
+        log.debug("start-init-start time: {}", initStartMs);
         try {
             super.initialize(name, conf);
             this.setConf(conf);
@@ -150,7 +151,7 @@ public void initialize(URI name, Configuration conf) throws IOException {
                 log.error(errMsg);
                 throw new IOException(errMsg);
             }
-            log.info("init jar, [elapse-ms: {}]", System.currentTimeMillis() - initJarStartMs);
+            log.debug("init jar, [elapse-ms: {}]", System.currentTimeMillis() - initJarStartMs);
 
             this.actualImplFS = jarLoader.getActualFileSystem();
             if (this.actualImplFS == null) {
@@ -160,7 +161,7 @@ public void initialize(URI name, Configuration conf) throws IOException {
 
             long actualInitStartMs = System.currentTimeMillis();
             this.actualImplFS.initialize(name, conf);
-            log.info("init actual file system, [elapse-ms: {}]", System.currentTimeMillis() - actualInitStartMs);
+            log.debug("init actual file system, [elapse-ms: {}]", System.currentTimeMillis() - actualInitStartMs);
             this.uri = this.actualImplFS.getUri();
             this.workingDir = this.actualImplFS.getWorkingDirectory();
         } catch (IOException ioe) {
@@ -170,7 +171,7 @@ public void initialize(URI name, Configuration conf) throws IOException {
             log.error("initialize failed! a unexpected exception occur!", e);
             throw new IOException("initialize failed! oops! a unexpected exception occur! " + e.toString(), e);
         }
-        log.info("total init file system, [elapse-ms: {}]", System.currentTimeMillis() - initStartMs);
+        log.debug("total init file system, [elapse-ms: {}]", System.currentTimeMillis() - initStartMs);
     }
 
 
@@ -694,7 +695,7 @@ public void close() throws IOException {
         try {
             long actualCloseStartTimeMs = System.currentTimeMillis();
             this.actualImplFS.close();
-            log.info("actual-file-system-close usedTime: {}", System.currentTimeMillis() - actualCloseStartTimeMs);
+            log.debug("actual-file-system-close usedTime: {}", System.currentTimeMillis() - actualCloseStartTimeMs);
             super.close();
         } catch (IOException ioe) {
             log.error("close fileSystem occur a ioException!", ioe);
@@ -704,6 +705,6 @@ public void close() throws IOException {
             throw new IOException("close fileSystem failed! a unexpected exception occur! " + e.getMessage());
         }
         long endMs = System.currentTimeMillis();
-        log.info("end-close time: {}, total-used-time: {}", endMs, endMs - initStartMs);
+        log.debug("end-close time: {}, total-used-time: {}", endMs, endMs - initStartMs);
     }
 }
diff --git a/src/main/java/com/qcloud/chdfs/fs/CHDFSHadoopFileSystemJarLoader.java b/src/main/java/com/qcloud/chdfs/fs/CHDFSHadoopFileSystemJarLoader.java
@@ -124,7 +124,7 @@ private boolean queryJarPluginInfo(String mountPointAddr, long appid, int jarPlu
             } else {
                 this.jarMd5 = jarInfoJson.get("JarMd5").getAsString();
             }
-            log.info("query jarPluginInfo, usedTimeMs: {}", (System.nanoTime() - startTimeNs) * 1.0 / 1000000);
+            log.debug("query jarPluginInfo, usedTimeMs: {}", (System.nanoTime() - startTimeNs) * 1.0 / 1000000);
         } catch (IOException e) {
             log.error("queryJarPluginInfo occur an io exception", e);
             return false;
@@ -141,8 +141,8 @@ synchronized boolean init(String mountPointAddr, long appid, int jarPluginServer
             if (!queryJarPluginInfo(mountPointAddr, appid, jarPluginServerPort, jarPluginServerHttps)) {
                 return false;
             }
-            log.info("query jar plugin info usedMs: {}", System.currentTimeMillis() - queryStartMs);
-            this.actualFileSystem = getAlreadyLoadedClassInfo(this.jarPath, this.versionId, this.jarMd5, tmpDirPath);
+            log.debug("query jar plugin info usedMs: {}", System.currentTimeMillis() - queryStartMs);
+            this.actualFileSystem = getAlreadyLoadedClassInfo(this.getClass().getClassLoader(), this.jarPath, this.versionId, this.jarMd5, tmpDirPath);
             if (this.actualFileSystem == null) {
                 return false;
             }
@@ -158,7 +158,7 @@ public void setChdfsDataTransferEndpointSuffix(String chdfsDataTransferEndpointS
         this.chdfsDataTransferEndpointSuffix = chdfsDataTransferEndpointSuffix;
     }
 
-    private static synchronized FileSystem getAlreadyLoadedClassInfo(String jarPath, String versionId, String jarMd5, String tmpDirPath) {
+    private static synchronized FileSystem getAlreadyLoadedClassInfo(ClassLoader currentClassLoader, String jarPath, String versionId, String jarMd5, String tmpDirPath) {
         if (alreadyLoadedFileSystemInfo != null
                 && alreadyLoadedFileSystemInfo.jarPath.equals(jarPath)
                 && alreadyLoadedFileSystemInfo.versionId.equals(versionId)
@@ -179,8 +179,7 @@ private static synchronized FileSystem getAlreadyLoadedClassInfo(String jarPath,
             log.error("get jar url failed.", e);
             return null;
         }
-        URLClassLoader chdfsJarClassLoader = new URLClassLoader(new URL[]{jarUrl},
-                ClassLoader.getSystemClassLoader());
+        URLClassLoader chdfsJarClassLoader = new URLClassLoader(new URL[]{jarUrl}, currentClassLoader);
         final String className = String.format(
                 "chdfs.%s.com.qcloud.chdfs.fs.CHDFSHadoopFileSystem", versionId);
         try {