[BitSail-465][Test] Fix NullPointerException of testCreateGenericExecutor method

bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java

@@ -33,6 +33,7 @@
 
 import java.io.IOException;
 import java.time.Duration;
+import java.util.HashSet;
 import java.util.List;
 import java.util.stream.Stream;
 
@@ -56,6 +57,7 @@ public class GenericExecutor extends AbstractExecutor {
 
   public GenericExecutor(GenericExecutorSetting setting) {
     this.setting = setting;
+    super.transferableFiles = new HashSet<>(setting.getAdditionalFiles());
   }
 
   @Override
@@ -81,7 +83,7 @@ public void init() {
             DockerLoggerFactory.getLogger(setting.getExecutorImage())).withSeparateOutputStreams())
         .withStartupAttempts(1)
         .withWorkingDirectory(EXECUTOR_ROOT_DIR.toAbsolutePath().toString())
-        .withCommand("bash", "-c", String.join(" ;", initCommands))
+        .withCommand("sh", "-c", String.join(" ;", initCommands))
         .waitingFor(new LogMessageWaitStrategy()
             .withRegEx(".*" + EXECUTOR_READY_MSG + ".*")
             .withStartupTimeout(Duration.ofSeconds(EXECUTOR_READY_TIMEOUT)));
@@ -104,7 +106,7 @@ public int run(String testId) throws Exception {
             + "============================================\n",
         testId, getContainerName());
 
-    Container.ExecResult result = executor.execInContainer("bash", "-c", commands);
+    Container.ExecResult result = executor.execInContainer("sh", "-c", commands);
 
     String stdOut = result.getStdout();
     String stdErr = result.getStderr();
@@ -120,7 +122,7 @@ public int run(String testId) throws Exception {
 
     if (exitCode != 0 && CollectionUtils.isNotEmpty(setting.getFailureHandleCommands())) {
       commands = String.join(" ", setting.getFailureHandleCommands());
-      result = executor.execInContainer("bash", "-c", commands);
+      result = executor.execInContainer("sh", "-c", commands);
 
       LOG.error("Job exited with code {}, will execute the failure handle commands now:\n"
               + "=========== FAILURE HANDLE COMMANDS ============\n"

bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorSettingTest.java

@@ -37,7 +37,7 @@ public void testReadSetting() throws Exception {
     GenericExecutorSetting setting = GenericExecutorSetting.initFromFile(settingFilePath);
 
     Assert.assertEquals("test-executor", setting.getExecutorName());
-    Assert.assertEquals("test-image", setting.getExecutorImage());
+    Assert.assertEquals("testcontainers/ryuk", setting.getExecutorImage());
 
     List<TransferableFile> transferableFileList = setting.getAdditionalFiles();
     Assert.assertEquals(2, transferableFileList.size());
@@ -46,9 +46,9 @@ public void testReadSetting() throws Exception {
     Assert.assertTrue(transferableFileList.contains(file1));
     Assert.assertTrue(transferableFileList.contains(file2));
 
-    Assert.assertEquals("pwd && sleep 5000",
+    Assert.assertEquals("pwd && sleep 1",
         String.join(" ", setting.getExecCommands()));
-    Assert.assertEquals("pwd && sleep 1000",
+    Assert.assertEquals("pwd && sleep 2",
         String.join(" ", setting.getFailureHandleCommands()));
 
     BitSailConfiguration globalJobConf = setting.getGlobalJobConf();

bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorTest.java

@@ -27,32 +27,50 @@
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.File;
 import java.nio.file.Paths;
-import java.util.Set;
+import java.util.List;
+import java.util.stream.Collectors;
 
 public class GenericExecutorTest {
+  public String transformHostFilePath(String hostFilePath) {
+    File module = Paths.get(AbstractExecutor.getLocalRootDir(),
+            "bitsail-test",
+            "bitsail-test-end-to-end",
+            "bitsail-test-e2e-base").toFile();
+    File jarFolder = Paths.get(module.getAbsolutePath(),
+            "src", "test", "resources", "executor", "generic", "host_file").toFile();
+    return Paths.get(
+            jarFolder.getAbsolutePath(),
+            Paths.get(hostFilePath).getFileName().toString()
+    ).toString();
+  }
 
   @Test
   public void testCreateGenericExecutor() throws Exception {
     String settingFilePath = Paths.get(GenericExecutorSettingTest.class.getClassLoader()
         .getResource("executor/generic/TestGenericExecutorSetting.json")
         .toURI()).toString();
 
-    GenericExecutor executor = new GenericExecutor(GenericExecutorSetting.initFromFile(settingFilePath));
+    GenericExecutorSetting executorSetting = GenericExecutorSetting.initFromFile(settingFilePath);
+    List<TransferableFile> collect = executorSetting.getAdditionalFiles()
+            .stream()
+            .map(x -> new TransferableFile(transformHostFilePath(x.getHostPath()), x.getContainerPath()))
+            .collect(Collectors.toList());
+    executorSetting.setAdditionalFiles(collect);
+    GenericExecutor executor = new GenericExecutor(executorSetting);
     Assert.assertEquals("test-executor", executor.getContainerName());
 
     BitSailConfiguration jobConf = BitSailConfiguration.newDefault();
     jobConf.set(ReaderOptions.READER_CLASS, FakeSource.class.getName());
     jobConf.set(WriterOptions.WRITER_CLASS, PrintSink.class.getName());
     executor.configure(jobConf);
-    Set<TransferableFile> transferableFiles = executor.getTransferableFiles();
-
-    String localRootPath = AbstractExecutor.getLocalRootDir();
-    TransferableFile file1 = new TransferableFile(Paths.get(localRootPath, "/local/1.jar").toAbsolutePath().toString(),
-        "/executor/1.jar");
-    TransferableFile file2 = new TransferableFile(Paths.get(localRootPath, "/local/2.jar").toAbsolutePath().toString(),
-        "/executor/2.jar");
-    Assert.assertTrue(transferableFiles.contains(file1));
-    Assert.assertTrue(transferableFiles.contains(file2));
+    executor.addJobConf(jobConf);
+    executor.init();
+
+    int exitCode = executor.run("testID");
+    Assert.assertEquals(exitCode, 0);
+
+    executor.close();
   }
 }

bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/TestGenericExecutorSetting.json

@@ -1,11 +1,11 @@
 {
   "name": "test-executor",
-  "executor-image": "test-image",
+  "executor-image": "testcontainers/ryuk",
   "exec-commands": [
-    "pwd", "&&", "sleep 5000"
+    "pwd", "&&", "sleep 1"
   ],
   "failure-handle-commands": [
-    "pwd", "&&", "sleep 1000"
+    "pwd", "&&", "sleep 2"
   ],
   "global-job-config": "{\"job\":{\"reader\":{\"reader_parallelism_num\":1},\"writer\":{\"writer_parallelism_num\":2}}}",
   "additional-files": [

bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/1.jar

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/2.jar

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */