[feat_863][taier-worker-plugin] spark sql proxy auto upload

DTStack · Nov 15, 2022 · b546cdb · b546cdb
1 parent e326209
commit b546cdb
Showing 1 changed file with 8 additions and 13 deletions.
diff --git a/...-core/src/main/java/com/dtstack/taier/sparkyarn/sparkyarn/file/SparkResourceUploader.java b/...-core/src/main/java/com/dtstack/taier/sparkyarn/sparkyarn/file/SparkResourceUploader.java
@@ -64,11 +64,11 @@ public SparkResourceUploader(
     }
 
     public void uploadSparkResource() {
-        String sparkResourcesDirProp = sparkExtProp.get(SparkYarnResourceInfo.SPARK_RESOURCES_DIR).toString();
-        if (StringUtils.isBlank(sparkResourcesDirProp)) {
+        Object sparkResourcesDirProp = sparkExtProp.get(SparkYarnResourceInfo.SPARK_RESOURCES_DIR);
+        if (sparkResourcesDirProp == null || StringUtils.isBlank(sparkResourcesDirProp.toString())) {
             sparkResourcesDirProp = SparkYarnResourceInfo.DEFAULT_SPARK_RESOURCES_DIR;
         }
-        final String sparkResourcesDir = sparkResourcesDirProp;
+        final String sparkResourcesDir = sparkResourcesDirProp.toString();
         String md5sum = sparkYarnConfig.getMd5sum();
         String sparkClearResourceRate =
                 sparkExtProp
@@ -125,20 +125,15 @@ private void uploadSparkSqlProxy(FileSystem fileSystem, String sparkResourcesDir
     private String getSqlProxyJarPath() {
         String path = this.getClass().getProtectionDomain().getCodeSource().getLocation().getPath();
 
-        File pluginDir = new File(path).getParentFile().getParentFile();
-        File[] sqlProxyDirs =
+        File pluginDir = new File(path).getParentFile();
+        File[] sqlProxyJars =
                 pluginDir.listFiles(
                         (dir, name) ->
                                 dir.isDirectory()
                                         && name.toLowerCase().startsWith("spark-sql-proxy"));
-        if (sqlProxyDirs != null && sqlProxyDirs.length == 1) {
-            File[] sqlProxyJars =
-                    sqlProxyDirs[0].listFiles(
-                            (dir, name) ->
-                                    name.toLowerCase().startsWith("spark-sql-proxy")
-                                            && name.toLowerCase().endsWith(".jar"));
-
-            if (sqlProxyJars != null && sqlProxyJars.length == 1) {
+        if (sqlProxyJars != null && sqlProxyJars.length == 1) {
+            String sqlProxyJar = sqlProxyJars[0].getName();
+            if (sqlProxyJar.toLowerCase().startsWith("spark-sql-proxy") && sqlProxyJar.toLowerCase().endsWith(".jar")) {
                 return sqlProxyJars[0].getAbsolutePath();
             }
         }