feat: merge 090 features to main

java/openmldb-batch/pom.xml

@@ -167,7 +167,11 @@
  </exclusion>
  </exclusions>
  </dependency>
-
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.4.14</version>
+ </dependency>
  <dependency>
  <groupId>org.apache.curator</groupId>
  <artifactId>curator-framework</artifactId>
@@ -182,6 +186,12 @@
  <groupId>org.apache.curator</groupId>
  <artifactId>curator-recipes</artifactId>
  <version>4.2.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
  </dependency>
 
  <!-- OpenMLDB -->
@@ -241,6 +251,7 @@
  <groupId>org.apache.hadoop</groupId>
  <artifactId>hadoop-aws</artifactId>
  <version>${hadoop.version}</version>
+ <scope>provided</scope>
  </dependency>
  </dependencies>
 

java/openmldb-common/pom.xml

@@ -40,6 +40,12 @@
  <groupId>org.apache.curator</groupId>
  <artifactId>curator-recipes</artifactId>
  <version>4.2.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
  </dependency>
  <dependency>
  <groupId>org.testng</groupId>

java/openmldb-taskmanager/pom.xml

@@ -134,6 +134,12 @@
  <groupId>org.apache.curator</groupId>
  <artifactId>curator-recipes</artifactId>
  <version>4.2.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
  </dependency>
  <dependency>
  <groupId>org.projectlombok</groupId>
@@ -142,9 +148,6 @@
  <scope>provided</scope>
  </dependency>
 
-
-
-
  <!-- Kubernetes -->
  <dependency>
  <groupId>io.fabric8</groupId>

java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/JobResultSaver.java

@@ -53,7 +53,7 @@
  */
 @Slf4j
 public class JobResultSaver {
- private static final Log log = LogFactory.getLog(JobResultSaver.class);
+ private static final Log logger = LogFactory.getLog(JobResultSaver.class);
 
  // false: unused, true: using
  // 0: unused, 1: saving, 2: finished but still in use
@@ -92,8 +92,8 @@
  public boolean saveFile(int resultId, String jsonData) {
  // No need to wait, cuz id status must have been changed by genResultId before.
  // It's a check.
- if (log.isDebugEnabled()) {
- log.debug("save result " + resultId + ", data " + jsonData);
+ if (logger.isDebugEnabled()) {
+ logger.debug("save result " + resultId + ", data " + jsonData);
  }
  int status = idStatus.get(resultId);
  if (status != 1) {
@@ -105,7 +105,7 @@
  idStatus.set(resultId, 2);
  idStatus.notifyAll();
  }
- log.info("saved all result of result " + resultId);
+ logger.info("saved all result of result " + resultId);
  return true;
  }
  // save to <log path>/tmp_result/<result_id>/<unique file name>
@@ -114,7 +114,7 @@
  File saveP = new File(savePath);
  if (!saveP.exists()) {
  boolean res = saveP.mkdirs();
- log.info("create save path " + savePath + ", status " + res);
+ logger.info("create save path " + savePath + ", status " + res);
  }
  }
  String fileFullPath = String.format("%s/%s", savePath, genUniqueFileName());
@@ -125,7 +125,7 @@
  + fileFullPath);
  }
  } catch (IOException e) {
- log.error("create file failed, path " + fileFullPath, e);
+ logger.error("create file failed, path " + fileFullPath, e);
  return false;
  }
 
@@ -135,7 +135,7 @@
  } catch (IOException e) {
  // Write failed, we'll lost a part of result, but it's ok for show sync job
  // output. So we just log it, and response the http request.
- log.error("write result to file failed, path " + fileFullPath, e);
+ logger.error("write result to file failed, path " + fileFullPath, e);
  return false;
  }
  return true;
@@ -151,7 +151,7 @@
  }
  }
  if (idStatus.get(resultId) != 2) {
- log.warn("read result timeout, result saving may be still running, try read anyway, id " + resultId);
+ logger.warn("read result timeout, result saving may be still running, try read anyway, id " + resultId);
  }
  String output = "";
  // all finished, read csv from savePath
@@ -163,7 +163,7 @@
  output = printFilesTostr(savePath);
  FileUtils.forceDelete(saveP);
  } else {
- log.info("empty result for " + resultId + ", show empty string");
+ logger.info("empty result for " + resultId + ", show empty string");
  }
  // reset id
  synchronized (idStatus) {
@@ -189,7 +189,7 @@
  }
  return stringWriter.toString();
  } catch (Exception e) {
- log.warn("read result met exception when read " + fileDir + ", " + e.getMessage());
+ logger.warn("read result met exception when read " + fileDir + ", " + e.getMessage());
  e.printStackTrace();
  return "read met exception, check the taskmanager log";
  }
@@ -219,7 +219,7 @@
  csvPrinter.printRecord(iter.next());
  }
  } catch (Exception e) {
- log.warn("error when print result file " + file + ", ignore it");
+ logger.warn("error when print result file " + file + ", ignore it");
  e.printStackTrace();
  }
  }

java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/zk/RecoverableZooKeeper.java

@@ -62,7 +62,7 @@ public class RecoverableZooKeeper {
  private final String quorumServers;
  private final int maxMultiSize; // unused now
 
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE", justification = "None. Its always been this way.")
+ //@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE", justification = "None. Its always been this way.")
  public RecoverableZooKeeper(String quorumServers, int sessionTimeout, Watcher watcher) throws IOException {
  // TODO: Add support for zk 'chroot'; we don't add it to the quorumServers
  // String as we should.