From 671c240bdb18b0c6e0c197041d18ac54ddb2b814 Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 11 Apr 2023 11:12:40 +0800 Subject: [PATCH 1/4] [BitSail-465][Fix]: fix NullPointerException of function testCreateGenericExecutor --- .../e2e/executor/generic/GenericExecutor.java | 8 ++-- .../generic/GenericExecutorSettingTest.java | 6 +-- .../executor/generic/GenericExecutorTest.java | 40 ++++++++++++++----- .../generic/TestGenericExecutorSetting.json | 6 +-- .../executor/generic/host_file/1.jar | 0 .../executor/generic/host_file/2.jar | 0 6 files changed, 40 insertions(+), 20 deletions(-) create mode 100644 bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/1.jar create mode 100644 bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/2.jar diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java index 10a109c15..9314c950f 100644 --- a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java +++ b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.time.Duration; +import java.util.HashSet; import java.util.List; import java.util.stream.Stream; @@ -61,6 +62,7 @@ public GenericExecutor(GenericExecutorSetting setting) { @Override public void configure(BitSailConfiguration executorConf) { this.conf = executorConf; + super.transferableFiles = new HashSet<>(setting.getAdditionalFiles()); } @Override @@ -81,7 +83,7 @@ public void init() { DockerLoggerFactory.getLogger(setting.getExecutorImage())).withSeparateOutputStreams()) .withStartupAttempts(1) .withWorkingDirectory(EXECUTOR_ROOT_DIR.toAbsolutePath().toString()) - .withCommand("bash", "-c", String.join(" ;", initCommands)) + .withCommand("sh", "-c", String.join(" ;", initCommands)) .waitingFor(new LogMessageWaitStrategy() .withRegEx(".*" + EXECUTOR_READY_MSG + ".*") .withStartupTimeout(Duration.ofSeconds(EXECUTOR_READY_TIMEOUT))); @@ -104,7 +106,7 @@ public int run(String testId) throws Exception { + "============================================\n", testId, getContainerName()); - Container.ExecResult result = executor.execInContainer("bash", "-c", commands); + Container.ExecResult result = executor.execInContainer("sh", "-c", commands); String stdOut = result.getStdout(); String stdErr = result.getStderr(); @@ -120,7 +122,7 @@ public int run(String testId) throws Exception { if (exitCode != 0 && CollectionUtils.isNotEmpty(setting.getFailureHandleCommands())) { commands = String.join(" ", setting.getFailureHandleCommands()); - result = executor.execInContainer("bash", "-c", commands); + result = executor.execInContainer("sh", "-c", commands); LOG.error("Job exited with code {}, will execute the failure handle commands now:\n" + "=========== FAILURE HANDLE COMMANDS ============\n" diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorSettingTest.java b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorSettingTest.java index c3353031e..03eb99370 100644 --- a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorSettingTest.java +++ b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorSettingTest.java @@ -37,7 +37,7 @@ public void testReadSetting() throws Exception { GenericExecutorSetting setting = GenericExecutorSetting.initFromFile(settingFilePath); Assert.assertEquals("test-executor", setting.getExecutorName()); - Assert.assertEquals("test-image", setting.getExecutorImage()); + Assert.assertEquals("testcontainers/ryuk", setting.getExecutorImage()); List transferableFileList = setting.getAdditionalFiles(); Assert.assertEquals(2, transferableFileList.size()); @@ -46,9 +46,9 @@ public void testReadSetting() throws Exception { Assert.assertTrue(transferableFileList.contains(file1)); Assert.assertTrue(transferableFileList.contains(file2)); - Assert.assertEquals("pwd && sleep 5000", + Assert.assertEquals("pwd && sleep 1", String.join(" ", setting.getExecCommands())); - Assert.assertEquals("pwd && sleep 1000", + Assert.assertEquals("pwd && sleep 2", String.join(" ", setting.getFailureHandleCommands())); BitSailConfiguration globalJobConf = setting.getGlobalJobConf(); diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorTest.java b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorTest.java index 97f3c1989..ca8f955dc 100644 --- a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorTest.java +++ b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutorTest.java @@ -27,10 +27,24 @@ import org.junit.Assert; import org.junit.Test; +import java.io.File; import java.nio.file.Paths; -import java.util.Set; +import java.util.List; +import java.util.stream.Collectors; public class GenericExecutorTest { + public String transformHostFilePath(String hostFilePath) { + File module = Paths.get(AbstractExecutor.getLocalRootDir(), + "bitsail-test", + "bitsail-test-end-to-end", + "bitsail-test-e2e-base").toFile(); + File jarFolder = Paths.get(module.getAbsolutePath(), + "src", "test", "resources", "executor", "generic", "host_file").toFile(); + return Paths.get( + jarFolder.getAbsolutePath(), + Paths.get(hostFilePath).getFileName().toString() + ).toString(); + } @Test public void testCreateGenericExecutor() throws Exception { @@ -38,21 +52,25 @@ public void testCreateGenericExecutor() throws Exception { .getResource("executor/generic/TestGenericExecutorSetting.json") .toURI()).toString(); - GenericExecutor executor = new GenericExecutor(GenericExecutorSetting.initFromFile(settingFilePath)); + GenericExecutorSetting executorSetting = GenericExecutorSetting.initFromFile(settingFilePath); + List collect = executorSetting.getAdditionalFiles() + .stream() + .map(x -> new TransferableFile(transformHostFilePath(x.getHostPath()), x.getContainerPath())) + .collect(Collectors.toList()); + executorSetting.setAdditionalFiles(collect); + GenericExecutor executor = new GenericExecutor(executorSetting); Assert.assertEquals("test-executor", executor.getContainerName()); BitSailConfiguration jobConf = BitSailConfiguration.newDefault(); jobConf.set(ReaderOptions.READER_CLASS, FakeSource.class.getName()); jobConf.set(WriterOptions.WRITER_CLASS, PrintSink.class.getName()); executor.configure(jobConf); - Set transferableFiles = executor.getTransferableFiles(); - - String localRootPath = AbstractExecutor.getLocalRootDir(); - TransferableFile file1 = new TransferableFile(Paths.get(localRootPath, "/local/1.jar").toAbsolutePath().toString(), - "/executor/1.jar"); - TransferableFile file2 = new TransferableFile(Paths.get(localRootPath, "/local/2.jar").toAbsolutePath().toString(), - "/executor/2.jar"); - Assert.assertTrue(transferableFiles.contains(file1)); - Assert.assertTrue(transferableFiles.contains(file2)); + executor.addJobConf(jobConf); + executor.init(); + + int exitCode = executor.run("testID"); + Assert.assertEquals(exitCode, 0); + + executor.close(); } } diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/TestGenericExecutorSetting.json b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/TestGenericExecutorSetting.json index fdf8b8220..4022ed90f 100644 --- a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/TestGenericExecutorSetting.json +++ b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/TestGenericExecutorSetting.json @@ -1,11 +1,11 @@ { "name": "test-executor", - "executor-image": "test-image", + "executor-image": "testcontainers/ryuk", "exec-commands": [ - "pwd", "&&", "sleep 5000" + "pwd", "&&", "sleep 1" ], "failure-handle-commands": [ - "pwd", "&&", "sleep 1000" + "pwd", "&&", "sleep 2" ], "global-job-config": "{\"job\":{\"reader\":{\"reader_parallelism_num\":1},\"writer\":{\"writer_parallelism_num\":2}}}", "additional-files": [ diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/1.jar b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/1.jar new file mode 100644 index 000000000..e69de29bb diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/2.jar b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/2.jar new file mode 100644 index 000000000..e69de29bb From 0dc1a4727cd5868911f3a23796bb6b83acf9728c Mon Sep 17 00:00:00 2001 From: Jake-00 <60967034+Jake-00@users.noreply.github.com> Date: Tue, 11 Apr 2023 14:29:04 +0800 Subject: [PATCH 2/4] Update 1.jar --- .../resources/executor/generic/host_file/1.jar | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/1.jar b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/1.jar index e69de29bb..034684fd6 100644 --- a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/1.jar +++ b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/1.jar @@ -0,0 +1,15 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ From 4de74893a29e480fbc863f351d5d0d1da8126e81 Mon Sep 17 00:00:00 2001 From: Jake-00 <60967034+Jake-00@users.noreply.github.com> Date: Tue, 11 Apr 2023 14:29:25 +0800 Subject: [PATCH 3/4] Update 2.jar --- .../resources/executor/generic/host_file/2.jar | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/2.jar b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/2.jar index e69de29bb..034684fd6 100644 --- a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/2.jar +++ b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/host_file/2.jar @@ -0,0 +1,15 @@ +/* + * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ From ab0f875b2c703a87a9083ebd88d2ec2b411b4dec Mon Sep 17 00:00:00 2001 From: Jake-00 <60967034+Jake-00@users.noreply.github.com> Date: Tue, 11 Apr 2023 14:31:59 +0800 Subject: [PATCH 4/4] Update GenericExecutor.java --- .../bitsail/test/e2e/executor/generic/GenericExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java index 9314c950f..586395fa9 100644 --- a/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java +++ b/bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/java/com/bytedance/bitsail/test/e2e/executor/generic/GenericExecutor.java @@ -57,12 +57,12 @@ public class GenericExecutor extends AbstractExecutor { public GenericExecutor(GenericExecutorSetting setting) { this.setting = setting; + super.transferableFiles = new HashSet<>(setting.getAdditionalFiles()); } @Override public void configure(BitSailConfiguration executorConf) { this.conf = executorConf; - super.transferableFiles = new HashSet<>(setting.getAdditionalFiles()); } @Override