[feat_spark320][taier-all] support spark 3.2.0 version

pom.xml

@@ -206,6 +206,11 @@
  <id>oss-sonatype</id>
  <url>https://oss.sonatype.org/content/repositories/releases/</url>
  </repository>
+
+ <repository>
+ <id>huawei</id>
+ <url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+ </repository>
  </repositories>
 
  <build>
@@ -283,7 +288,7 @@
  <configuration>
  <aggregate>true</aggregate>
  <reportOutputDirectory>javadocs</reportOutputDirectory>
- <destDir>engine-java-docs</destDir>
+ <destDir>taier-java-docs</destDir>
  </configuration>
  </plugin>
  </plugins>

taier-common/pom.xml

@@ -222,7 +222,7 @@
  <version>3.0.1</version>
  <configuration>
  <reportOutputDirectory>../javadocs</reportOutputDirectory>
- <destDir>engine-api-client</destDir>
+ <destDir>taier-api-client</destDir>
  <additionalJOptions>
  <additionalJOption>-Xdoclint:none</additionalJOption>
  </additionalJOptions>

taier-data-develop/pom.xml

@@ -131,7 +131,7 @@
  <plugin>
  <groupId>org.apache.maven.plugins</groupId>
  <artifactId>maven-shade-plugin</artifactId>
- <version>2.4.3</version>
+ <version>3.2.4</version>
  <dependencies>
  <dependency>
  <groupId>org.springframework.boot</groupId>

taier-data-develop/src/main/java/com/dtstack/taier/develop/service/develop/impl/DevelopSelectSqlService.java

@@ -150,6 +150,7 @@ public String sendSqlTask(String sql, String taskParams, String jobId, Task task
  paramActionExt.setTenantId(task.getTenantId());
  paramActionExt.setQueueName(task.getQueueName());
  paramActionExt.setDatasourceId(task.getDatasourceId());
+ paramActionExt.setComponentVersion(task.getComponentVersion());
  actionService.start(paramActionExt);
  return jobId;
  }

taier-data-develop/src/main/java/com/dtstack/taier/develop/service/develop/saver/AbstractTaskSaver.java

@@ -21,6 +21,7 @@
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.dtstack.taier.common.enums.EComponentType;
+import com.dtstack.taier.common.enums.EComputeType;
 import com.dtstack.taier.common.enums.EScheduleJobType;
 import com.dtstack.taier.common.enums.EScheduleStatus;
 import com.dtstack.taier.common.enums.ESubmitStatus;
@@ -183,6 +184,9 @@ public TaskVO updateTaskInfo(TaskResourceParam taskResourceParam) {
  Task task = developTaskService.getOne(Wrappers.lambdaQuery(Task.class)
  .eq(Task::getName, taskVO.getName())
  .eq(Task::getTenantId, taskVO.getTenantId()));
+ if(EComputeType.BATCH.getType() == taskVO.getComputeType()){
+ taskVO.setJobId(null);
+ }
 
  if (taskVO.getId() != null && taskVO.getId() > 0) {
  //update

taier-data-develop/src/main/java/com/dtstack/taier/develop/service/develop/saver/DefaultTaskSaver.java

@@ -26,6 +26,7 @@
 import com.dtstack.taier.develop.dto.devlop.TaskVO;
 import com.dtstack.taier.develop.service.develop.impl.DevelopTaskTaskService;
 import com.dtstack.taier.develop.service.user.UserService;
+import com.dtstack.taier.pluginapi.enums.ComputeType;
 import com.dtstack.taier.pluginapi.enums.EJobType;
 import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
@@ -50,13 +51,13 @@ public class DefaultTaskSaver extends AbstractTaskSaver {
  @Autowired
  private UserService userService;
 
- @Autowired
- private DevelopTaskTaskService developTaskTaskService;
 
  @Override
  public TaskResourceParam beforeProcessing(TaskResourceParam taskResourceParam) {
  // sql 任务必须选择数据源
  EScheduleJobType scheduleJobType = EScheduleJobType.getByTaskType(taskResourceParam.getTaskType());
+ taskResourceParam.setTaskParams(taskResourceParam.getTaskParams() == null ? taskTemplateService.getTaskTemplate(taskResourceParam.getTaskType(), taskResourceParam.getComponentVersion()).getParams() : taskResourceParam.getTaskParams());
+ taskResourceParam.setComputeType(ComputeType.BATCH.getType());
  if (EComputeType.BATCH.getType() == scheduleJobType.getComputeType().getType() && EJobType.SQL.getType() == scheduleJobType.getEngineJobType()) {
  if (null == taskResourceParam.getDatasourceId()) {
  throw new TaierDefineException(ErrorCode.DATA_SOURCE_NOT_SET);

taier-datasource/pom.xml

@@ -89,6 +89,24 @@
  </dependencies>
  </dependencyManagement>
 
+
+ <repositories>
+ <repository>
+ <id>central</id>
+ <url>https://repo1.maven.org/maven2/</url>
+ </repository>
+
+ <repository>
+ <id>oss-sonatype</id>
+ <url>https://oss.sonatype.org/content/repositories/releases/</url>
+ </repository>
+
+ <repository>
+ <id>huawei</id>
+ <url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+ </repository>
+ </repositories>
+
  <build>
  <plugins>
  <plugin>

taier-datasource/taier-datasource-api/src/main/java/com/dtstack/taier/datasource/api/base/ClientCache.java

@@ -25,11 +25,15 @@
 import com.dtstack.taier.datasource.api.client.IRestful;
 import com.dtstack.taier.datasource.api.client.ITable;
 import com.dtstack.taier.datasource.api.client.IYarn;
+import com.dtstack.taier.datasource.api.config.Configuration;
+import com.dtstack.taier.datasource.api.context.ClientEnvironment;
 import com.dtstack.taier.datasource.api.exception.InitializeException;
+import com.dtstack.taier.datasource.api.manager.ManagerFactory;
 import com.dtstack.taier.datasource.api.manager.list.ClientManager;
 import com.dtstack.taier.datasource.api.source.DataSourceType;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.HashMap;
 import java.util.Objects;
 
 /**
@@ -78,6 +82,18 @@ public static IClient getClient(Integer dataSourceType) {
  return getClientByType(IClient.class, dataSourceType);
  }
 
+ public static void main(String[] args) {
+ Configuration configuration = new Configuration(new HashMap<>());
+ ClientEnvironment clientEnvironment = new ClientEnvironment(configuration);
+ clientEnvironment.start();
+ ClientCache.setEnv(clientEnvironment.getManagerFactory().getManager(ClientManager.class));
+ ClientManager clientManager = new ClientManager();
+ clientManager.setManagerFactory(new ManagerFactory());
+ setEnv(clientManager);
+ IClient client = getClient(DataSourceType.KAFKA.getVal());
+ System.out.println(client);
+ }
+
  /**
  * 获取 HDFS 文件客户端
  *

taier-datasource/taier-datasource-plugin/taier-datasource-plugin-aws_s3/pom.xml

@@ -51,7 +51,7 @@
  <plugin>
  <groupId>org.apache.maven.plugins</groupId>
  <artifactId>maven-shade-plugin</artifactId>
- <version>3.2.1</version>
+ <version>3.2.4</version>
  <executions>
  <execution>
  <phase>package</phase>

taier-datasource/taier-datasource-plugin/taier-datasource-plugin-csp_s3/pom.xml

@@ -50,7 +50,7 @@
  <plugin>
  <groupId>org.apache.maven.plugins</groupId>
  <artifactId>maven-shade-plugin</artifactId>
- <version>3.2.1</version>
+ <version>3.2.4</version>
  <executions>
  <execution>
  <phase>package</phase>

taier-datasource/taier-datasource-plugin/taier-datasource-plugin-hive1/pom.xml

@@ -22,10 +22,10 @@
  <lzo-hadoop.version>1.0.5</lzo-hadoop.version>
  <parquet-hadoop.version>1.8.3</parquet-hadoop.version>
  <snappy-java.version>1.1.2.6</snappy-java.version>
- <hive-exec.version>1.1.1</hive-exec.version>
+ <hive-exec.version>1.2.1</hive-exec.version>
  <hadoop-client.version>2.7.3</hadoop-client.version>
  <orc-core.version>1.6.3</orc-core.version>
- <hive-jdbc.version>1.1.1</hive-jdbc.version>
+ <hive-jdbc.version>1.2.1</hive-jdbc.version>
  </properties>
 
  <dependencies>
@@ -424,6 +424,11 @@
  </exclusion>
  </exclusions>
  </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.3</version>
+ </dependency>
  </dependencies>
 
  <build>