Skip to content

Commit

Permalink
improve command execution
Browse files Browse the repository at this point in the history
  • Loading branch information
@mandoway
mandoway committed Oct 22, 2024
1 parent a75ec57 commit 9be1649
Showing 1 changed file with 79 additions and 102 deletions.
181 changes: 79 additions & 102 deletions study/base.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,23 @@
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-21T20:14:35.520488Z",
"start_time": "2024-10-21T20:14:34.916124Z"
"end_time": "2024-10-22T09:27:59.048215Z",
"start_time": "2024-10-22T09:27:58.153499Z"
}
},
"cell_type": "code",
"source": [
"%useLatestDescriptors\n",
"%use coroutines"
],
"outputs": [],
"execution_count": 1
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-22T09:28:02.144864Z",
"start_time": "2024-10-22T09:28:01.474419Z"
}
},
"cell_type": "code",
Expand All @@ -43,30 +58,30 @@
"\n",
"data class CommandOutput(val stdout: String, val stderr: String)\n",
"\n",
"fun String.runCommand(workingDir: File): CommandOutput? {\n",
" return try {\n",
" ProcessBuilder(*split(\" \").toTypedArray())\n",
" .directory(workingDir)\n",
" .redirectOutput(ProcessBuilder.Redirect.PIPE)\n",
" .redirectError(ProcessBuilder.Redirect.PIPE)\n",
" .start()\n",
" .also { it.waitFor(60, TimeUnit.MINUTES) }\n",
" .let {\n",
" CommandOutput(\n",
" stdout = it.inputStream.bufferedReader().readText(),\n",
" stderr = it.errorStream.bufferedReader().readText()\n",
" )\n",
" }\n",
" } catch (e: IOException) {\n",
" e.printStackTrace()\n",
" null\n",
"fun String.runCommand(workingDir: File): CommandOutput? = runBlocking(Dispatchers.IO) {\n",
" withTimeout(5.minutes) {\n",
" val proc = ProcessBuilder(*split(\" \").toTypedArray()).directory(workingDir).start()\n",
"\n",
" try {\n",
" val stdout = async { runInterruptible { proc.inputReader().use { it.readText() } } }\n",
" val stderr = async { runInterruptible { proc.errorReader().use { it.readText() } } }\n",
"\n",
" runInterruptible { proc.waitFor() }\n",
"\n",
" CommandOutput(stdout = stdout.await(), stderr = stderr.await())\n",
" } catch (e: IOException) {\n",
" e.printStackTrace()\n",
" null\n",
" } finally {\n",
" proc.destroy()\n",
" }\n",
" }\n",
"}\n",
"\n",
"fun String.runCommandInRoot() = runCommand(File(\"..\"))"
],
"outputs": [],
"execution_count": 5
"execution_count": 2
},
{
"metadata": {},
Expand All @@ -76,8 +91,8 @@
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-21T20:14:45.708359Z",
"start_time": "2024-10-21T20:14:39.350083Z"
"end_time": "2024-10-21T20:32:47.880068Z",
"start_time": "2024-10-21T20:32:41.766005Z"
}
},
"cell_type": "code",
Expand All @@ -92,12 +107,12 @@
"CommandOutput(stdout=, stderr=)"
]
},
"execution_count": 6,
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": 6
"execution_count": 27
},
{
"metadata": {},
Expand All @@ -107,13 +122,14 @@
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-21T20:26:04.542529Z",
"start_time": "2024-10-21T20:26:04.467967Z"
"end_time": "2024-10-22T09:28:21.795290Z",
"start_time": "2024-10-22T09:28:21.663664Z"
}
},
"cell_type": "code",
"source": [
"import java.nio.file.Path\n",
"import kotlin.io.path.div\n",
"import kotlin.io.path.pathString\n",
"\n",
"val Path.firstCueFile get() = this / listDirectoryEntries(glob = \"*.cue\").first().fileName\n",
Expand All @@ -123,13 +139,13 @@
"fun runSeruVulcanIn(studyDir: Path, instance: Path) = (seruBaseCommand(studyDir, instance) + \" -r vulcan\").runCommandInRoot()"
],
"outputs": [],
"execution_count": 21
"execution_count": 4
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-21T20:26:05.021738Z",
"start_time": "2024-10-21T20:26:04.872663Z"
"end_time": "2024-10-22T09:28:28.880891Z",
"start_time": "2024-10-22T09:28:28.597896Z"
}
},
"cell_type": "code",
Expand All @@ -138,6 +154,7 @@
"\n",
"val runsPerInstance = 5\n",
"val totalRuns = instances.toList().size * runsPerInstance\n",
"val studyPath = Path(\"study\")\n",
"\n",
"fun moveLatestSeruResultTo(dir: Path) = Path(\"..\")\n",
" .listDirectoryEntries(glob = \"seru*\")\n",
Expand All @@ -151,20 +168,21 @@
"\n",
"fun runAllInstances(outputDir: String, runCmd: (Path, Path) -> CommandOutput?) = instances\n",
" .forEachIndexed { index, instance ->\n",
" val instanceOutputPath = Path(outputDir) / instance\n",
" repeat(runsPerInstance) {\n",
" val instanceDir = (Path(outputDir) / instance / \"run_$it\").createDirectories()\n",
" val instanceDir = (instanceOutputPath / \"run_$it\").createDirectories()\n",
" val cur = it + (index * runsPerInstance)\n",
" println(\"$cur/$totalRuns - Start run for $instanceDir\")\n",
"\n",
" val output = runCmd(Path(\"study\"), instance)\n",
" val output = runCmd(studyPath, instance)\n",
"\n",
" moveLatestSeruResultTo(instanceDir)\n",
" saveLogsIfExisting(instanceDir, output)\n",
" }\n",
" }"
],
"outputs": [],
"execution_count": 22
"execution_count": 5
},
{
"metadata": {},
Expand All @@ -178,8 +196,8 @@
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-21T20:26:48.177750Z",
"start_time": "2024-10-21T20:26:17.599809Z"
"end_time": "2024-10-21T22:36:14.297496Z",
"start_time": "2024-10-21T21:16:19.464128Z"
}
},
"cell_type": "code",
Expand All @@ -189,80 +207,39 @@
"name": "stdout",
"output_type": "stream",
"text": [
"0/80 - Start run for results_perses/instances/extra/issue_2_inlined/final/run_0\n"
"0/80 - Start run for /Users/markus/Documents/UniProjects/MA/seru/study/results_perses/instances/extra/issue_2_inlined/final/run_0\n",
"1/80 - Start run for results_perses/instances/extra/issue_2_inlined/final/run_1\n",
"2/80 - Start run for results_perses/instances/extra/issue_2_inlined/final/run_2\n",
"3/80 - Start run for results_perses/instances/extra/issue_2_inlined/final/run_3\n",
"4/80 - Start run for results_perses/instances/extra/issue_2_inlined/final/run_4\n",
"5/80 - Start run for /Users/markus/Documents/UniProjects/MA/seru/study/results_perses/instances/panic/issue_2584/v1/run_0\n",
"6/80 - Start run for results_perses/instances/panic/issue_2584/v1/run_1\n",
"7/80 - Start run for results_perses/instances/panic/issue_2584/v1/run_2\n",
"8/80 - Start run for results_perses/instances/panic/issue_2584/v1/run_3\n",
"9/80 - Start run for results_perses/instances/panic/issue_2584/v1/run_4\n",
"10/80 - Start run for /Users/markus/Documents/UniProjects/MA/seru/study/results_perses/instances/panic/issue_2584/v2/run_0\n",
"11/80 - Start run for results_perses/instances/panic/issue_2584/v2/run_1\n",
"12/80 - Start run for results_perses/instances/panic/issue_2584/v2/run_2\n",
"13/80 - Start run for results_perses/instances/panic/issue_2584/v2/run_3\n",
"14/80 - Start run for results_perses/instances/panic/issue_2584/v2/run_4\n",
"15/80 - Start run for /Users/markus/Documents/UniProjects/MA/seru/study/results_perses/instances/panic/issue_2584/final/run_0\n",
"16/80 - Start run for results_perses/instances/panic/issue_2584/final/run_1\n",
"17/80 - Start run for results_perses/instances/panic/issue_2584/final/run_2\n",
"18/80 - Start run for results_perses/instances/panic/issue_2584/final/run_3\n",
"19/80 - Start run for results_perses/instances/panic/issue_2584/final/run_4\n",
"20/80 - Start run for /Users/markus/Documents/UniProjects/MA/seru/study/results_perses/instances/panic/issue_2490_inlined/v1/run_0\n",
"21/80 - Start run for results_perses/instances/panic/issue_2490_inlined/v1/run_1\n",
"22/80 - Start run for results_perses/instances/panic/issue_2490_inlined/v1/run_2\n"
]
},
{
"ename": "java.nio.file.FileAlreadyExistsException",
"evalue": "results_perses/instances/extra/issue_2_inlined/final/run_0/seru_result",
"ename": "org.jetbrains.kotlinx.jupyter.exceptions.ReplInterruptedException",
"evalue": "The execution was interrupted",
"output_type": "error",
"traceback": [
"java.nio.file.FileAlreadyExistsException: results_perses/instances/extra/issue_2_inlined/final/run_0/seru_result",
"\tat java.base/sun.nio.fs.UnixFileSystem.move(UnixFileSystem.java:907)",
"\tat java.base/sun.nio.fs.UnixFileSystemProvider.move(UnixFileSystemProvider.java:309)",
"\tat java.base/java.nio.file.Files.move(Files.java:1429)",
"\tat Line_23_jupyter.moveLatestSeruResultTo(Line_23.jupyter.kts:9) at Cell In[22], line 9",
"\tat Line_23_jupyter.runAllInstances(Line_23.jupyter.kts:25) at Cell In[22], line 25",
"\tat Line_25_jupyter.<init>(Line_25.jupyter.kts:1) at Cell In[24], line 1",
"\tat java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:67)",
"\tat java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500)",
"\tat java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:484)",
"\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.evalWithConfigAndOtherScriptsResults(BasicJvmScriptEvaluator.kt:122)",
"\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.invoke$suspendImpl(BasicJvmScriptEvaluator.kt:48)",
"\tat kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.invoke(BasicJvmScriptEvaluator.kt)",
"\tat kotlin.script.experimental.jvm.BasicJvmReplEvaluator.eval(BasicJvmReplEvaluator.kt:49)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.InternalEvaluatorImpl$eval$resultWithDiagnostics$1.invokeSuspend(InternalEvaluatorImpl.kt:133)",
"\tat kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)",
"\tat kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:104)",
"\tat kotlinx.coroutines.EventLoopImplBase.processNextEvent(EventLoop.common.kt:277)",
"\tat kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt:95)",
"\tat kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt:69)",
"\tat kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source)",
"\tat kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking$default(Builders.kt:48)",
"\tat kotlinx.coroutines.BuildersKt.runBlocking$default(Unknown Source)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.InternalEvaluatorImpl.eval(InternalEvaluatorImpl.kt:133)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl$execute$1$result$1.invoke(CellExecutorImpl.kt:80)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl$execute$1$result$1.invoke(CellExecutorImpl.kt:78)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.ReplForJupyterImpl.withHost(ReplForJupyterImpl.kt:742)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.CellExecutorImpl.execute-L4Nmkdk(CellExecutorImpl.kt:78)",
"\tat org.jetbrains.kotlinx.jupyter.repl.execution.CellExecutor$DefaultImpls.execute-L4Nmkdk$default(CellExecutor.kt:13)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.ReplForJupyterImpl.evaluateUserCode-wNURfNM(ReplForJupyterImpl.kt:565)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.ReplForJupyterImpl.evalExImpl(ReplForJupyterImpl.kt:423)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.ReplForJupyterImpl.access$evalExImpl(ReplForJupyterImpl.kt:139)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.ReplForJupyterImpl$evalEx$1.invoke(ReplForJupyterImpl.kt:416)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.ReplForJupyterImpl$evalEx$1.invoke(ReplForJupyterImpl.kt:415)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.ReplForJupyterImpl.withEvalContext(ReplForJupyterImpl.kt:396)",
"\tat org.jetbrains.kotlinx.jupyter.repl.impl.ReplForJupyterImpl.evalEx(ReplForJupyterImpl.kt:415)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor$processExecuteRequest$1$response$1$1.invoke(IdeCompatibleMessageRequestProcessor.kt:170)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor$processExecuteRequest$1$response$1$1.invoke(IdeCompatibleMessageRequestProcessor.kt:169)",
"\tat org.jetbrains.kotlinx.jupyter.streams.BlockingSubstitutionEngine.withDataSubstitution(SubstitutionEngine.kt:70)",
"\tat org.jetbrains.kotlinx.jupyter.streams.StreamSubstitutionManager.withSubstitutedStreams(StreamSubstitutionManager.kt:118)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor.withForkedIn(IdeCompatibleMessageRequestProcessor.kt:342)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor.access$withForkedIn(IdeCompatibleMessageRequestProcessor.kt:66)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor$evalWithIO$1$1.invoke(IdeCompatibleMessageRequestProcessor.kt:356)",
"\tat org.jetbrains.kotlinx.jupyter.streams.BlockingSubstitutionEngine.withDataSubstitution(SubstitutionEngine.kt:70)",
"\tat org.jetbrains.kotlinx.jupyter.streams.StreamSubstitutionManager.withSubstitutedStreams(StreamSubstitutionManager.kt:118)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor.withForkedErr(IdeCompatibleMessageRequestProcessor.kt:331)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor.access$withForkedErr(IdeCompatibleMessageRequestProcessor.kt:66)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor$evalWithIO$1.invoke(IdeCompatibleMessageRequestProcessor.kt:355)",
"\tat org.jetbrains.kotlinx.jupyter.streams.BlockingSubstitutionEngine.withDataSubstitution(SubstitutionEngine.kt:70)",
"\tat org.jetbrains.kotlinx.jupyter.streams.StreamSubstitutionManager.withSubstitutedStreams(StreamSubstitutionManager.kt:118)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor.withForkedOut(IdeCompatibleMessageRequestProcessor.kt:323)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor.evalWithIO(IdeCompatibleMessageRequestProcessor.kt:354)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor$processExecuteRequest$1$response$1.invoke(IdeCompatibleMessageRequestProcessor.kt:169)",
"\tat org.jetbrains.kotlinx.jupyter.messaging.IdeCompatibleMessageRequestProcessor$processExecuteRequest$1$response$1.invoke(IdeCompatibleMessageRequestProcessor.kt:168)",
"\tat org.jetbrains.kotlinx.jupyter.execution.JupyterExecutorImpl$Task.execute(JupyterExecutorImpl.kt:41)",
"\tat org.jetbrains.kotlinx.jupyter.execution.JupyterExecutorImpl$executorThread$1.invoke(JupyterExecutorImpl.kt:81)",
"\tat org.jetbrains.kotlinx.jupyter.execution.JupyterExecutorImpl$executorThread$1.invoke(JupyterExecutorImpl.kt:79)",
"\tat kotlin.concurrent.ThreadsKt$thread$thread$1.run(Thread.kt:30)",
"",
"java.nio.file.FileAlreadyExistsException: results_perses/instances/extra/issue_2_inlined/final/run_0/seru_result",
"at Cell In[22], line 9",
""
]
"traceback": []
}
],
"execution_count": 24
"execution_count": 31
},
{
"metadata": {},
Expand Down

0 comments on commit 9be1649

Please sign in to comment.