Skip to content

Commit ff3c8a8

Browse files
ileasileileasile
committed
Add code preprocessors as a part of library API
Fixes #223
1 parent 8494b17 commit ff3c8a8

File tree

15 files changed

+181
-65
lines changed

15 files changed

+181
-65
lines changed

‎gradle.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ logbackVersion=1.2.3
1616
http4kVersion=4.4.0.1
1717
artifactsPath=build/artifacts
1818

19-
baseVersion=0.9.1
19+
baseVersion=0.10.0
2020

2121
projectRepoUrl=https://github.com/Kotlin/kotlin-jupyter
2222
docsRepo=git@github.com:ileasile/kotlin-jupyter-docs.git

‎jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/CodePreprocessor.kt

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package org.jetbrains.kotlinx.jupyter.api
2+
3+
import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinitionProducer
4+
5+
/**
6+
* Preprocesses the cell code before its execution
7+
*/
8+
interface CodePreprocessor {
9+
/**
10+
* Returns `true` if this preprocessor accepts the given [code]
11+
*/
12+
fun accepts(code: String): Boolean = true
13+
14+
/**
15+
* Performs code preprocessing
16+
*/
17+
fun process(code: String, host: KotlinKernelHost): Result
18+
19+
data class Result(val code: Code, val libraries: List<LibraryDefinitionProducer> = emptyList())
20+
}

‎jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/JupyterIntegration.kt

+35-14
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ package org.jetbrains.kotlinx.jupyter.api.libraries
33
import org.jetbrains.kotlinx.jupyter.api.AfterCellExecutionCallback
44
import org.jetbrains.kotlinx.jupyter.api.ClassAnnotationHandler
55
import org.jetbrains.kotlinx.jupyter.api.ClassDeclarationsCallback
6+
import org.jetbrains.kotlinx.jupyter.api.Code
67
import org.jetbrains.kotlinx.jupyter.api.CodeCell
8+
import org.jetbrains.kotlinx.jupyter.api.CodePreprocessor
79
import org.jetbrains.kotlinx.jupyter.api.ExecutionCallback
810
import org.jetbrains.kotlinx.jupyter.api.FieldHandler
911
import org.jetbrains.kotlinx.jupyter.api.FieldHandlerByClass
@@ -54,6 +56,8 @@ abstract class JupyterIntegration : LibraryDefinitionProducer {
5456

5557
private val repositories = mutableListOf<String>()
5658

59+
private val codePreprocessors = mutableListOf<CodePreprocessor>()
60+
5761
fun addRenderer(handler: RendererTypeHandler) {
5862
renderers.add(handler)
5963
}
@@ -70,6 +74,10 @@ abstract class JupyterIntegration : LibraryDefinitionProducer {
7074
fileAnnotations.add(handler)
7175
}
7276

77+
fun addCodePreprocessor(preprocessor: CodePreprocessor) {
78+
codePreprocessors.add(preprocessor)
79+
}
80+
7381
inline fun <reified T : Any> render(noinline renderer: CodeCell.(T) -> Any) {
7482
return renderWithHost { _, value: T -> renderer(this, value) }
7583
}
@@ -152,21 +160,34 @@ abstract class JupyterIntegration : LibraryDefinitionProducer {
152160
addFileAnnotationHanlder(FileAnnotationHandler(T::class, callback))
153161
}
154162

163+
fun preprocessCodeWithLibraries(callback: KotlinKernelHost.(Code) -> CodePreprocessor.Result) {
164+
addCodePreprocessor(object : CodePreprocessor {
165+
override fun process(code: String, host: KotlinKernelHost): CodePreprocessor.Result {
166+
return host.callback(code)
167+
}
168+
})
169+
}
170+
171+
fun preprocessCode(callback: KotlinKernelHost.(Code) -> Code) {
172+
preprocessCodeWithLibraries { CodePreprocessor.Result(this.callback(it)) }
173+
}
174+
155175
internal fun getDefinition() =
156-
LibraryDefinitionImpl(
157-
init = init,
158-
renderers = renderers,
159-
converters = converters,
160-
imports = imports,
161-
dependencies = dependencies,
162-
repositories = repositories,
163-
initCell = beforeCellExecution,
164-
afterCellExecution = afterCellExecution,
165-
shutdown = shutdownCallbacks,
166-
classAnnotations = classAnnotations,
167-
fileAnnotations = fileAnnotations,
168-
resources = resources,
169-
)
176+
libraryDefinition {
177+
it.init = init
178+
it.renderers = renderers
179+
it.converters = converters
180+
it.imports = imports
181+
it.dependencies = dependencies
182+
it.repositories = repositories
183+
it.initCell = beforeCellExecution
184+
it.afterCellExecution = afterCellExecution
185+
it.shutdown = shutdownCallbacks
186+
it.classAnnotations = classAnnotations
187+
it.fileAnnotations = fileAnnotations
188+
it.resources = resources
189+
it.codePreprocessors = codePreprocessors
190+
}
170191
}
171192

172193
override fun getDefinitions(notebook: Notebook): List<LibraryDefinition> {

‎jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/LibraryDefinition.kt

+7
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.jetbrains.kotlinx.jupyter.api.libraries
22

33
import org.jetbrains.kotlinx.jupyter.api.AfterCellExecutionCallback
44
import org.jetbrains.kotlinx.jupyter.api.ClassAnnotationHandler
5+
import org.jetbrains.kotlinx.jupyter.api.CodePreprocessor
56
import org.jetbrains.kotlinx.jupyter.api.ExecutionCallback
67
import org.jetbrains.kotlinx.jupyter.api.FieldHandler
78
import org.jetbrains.kotlinx.jupyter.api.FileAnnotationHandler
@@ -85,6 +86,12 @@ interface LibraryDefinition {
8586
val resources: List<LibraryResource>
8687
get() = emptyList()
8788

89+
/**
90+
* List of code preprocessors
91+
*/
92+
val codePreprocessors: List<CodePreprocessor>
93+
get() = emptyList()
94+
8895
/**
8996
* Minimal kernel version that is supported by this library
9097
*/

‎jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/libraries/LibraryDefinitionImpl.kt

+34-19
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,45 @@ package org.jetbrains.kotlinx.jupyter.api.libraries
22

33
import org.jetbrains.kotlinx.jupyter.api.AfterCellExecutionCallback
44
import org.jetbrains.kotlinx.jupyter.api.ClassAnnotationHandler
5+
import org.jetbrains.kotlinx.jupyter.api.CodePreprocessor
56
import org.jetbrains.kotlinx.jupyter.api.ExecutionCallback
67
import org.jetbrains.kotlinx.jupyter.api.FieldHandler
78
import org.jetbrains.kotlinx.jupyter.api.FileAnnotationHandler
89
import org.jetbrains.kotlinx.jupyter.api.KotlinKernelVersion
910
import org.jetbrains.kotlinx.jupyter.api.RendererTypeHandler
1011

1112
/**
12-
* Trivial implementation of [LibraryDefinition].
13-
* You may use it in simple cases instead of overriding [LibraryDefinition]
14-
* to avoid additional anonymous classes creation
13+
* Trivial implementation of [LibraryDefinition] - simple container.
1514
*/
16-
class LibraryDefinitionImpl(
17-
override val dependencies: List<String> = emptyList(),
18-
override val repositories: List<String> = emptyList(),
19-
override val imports: List<String> = emptyList(),
20-
override val init: List<ExecutionCallback<*>> = emptyList(),
21-
override val initCell: List<ExecutionCallback<*>> = emptyList(),
22-
override val afterCellExecution: List<AfterCellExecutionCallback> = emptyList(),
23-
override val shutdown: List<ExecutionCallback<*>> = emptyList(),
24-
override val renderers: List<RendererTypeHandler> = emptyList(),
25-
override val converters: List<FieldHandler> = emptyList(),
26-
override val classAnnotations: List<ClassAnnotationHandler> = emptyList(),
27-
override val fileAnnotations: List<FileAnnotationHandler> = emptyList(),
28-
override val resources: List<LibraryResource> = emptyList(),
29-
override val minKernelVersion: KotlinKernelVersion? = null,
30-
override val originalDescriptorText: String? = null,
31-
) : LibraryDefinition
15+
class LibraryDefinitionImpl private constructor() : LibraryDefinition {
16+
override var dependencies: List<String> = emptyList()
17+
override var repositories: List<String> = emptyList()
18+
override var imports: List<String> = emptyList()
19+
override var init: List<ExecutionCallback<*>> = emptyList()
20+
override var initCell: List<ExecutionCallback<*>> = emptyList()
21+
override var afterCellExecution: List<AfterCellExecutionCallback> = emptyList()
22+
override var shutdown: List<ExecutionCallback<*>> = emptyList()
23+
override var renderers: List<RendererTypeHandler> = emptyList()
24+
override var converters: List<FieldHandler> = emptyList()
25+
override var classAnnotations: List<ClassAnnotationHandler> = emptyList()
26+
override var fileAnnotations: List<FileAnnotationHandler> = emptyList()
27+
override var resources: List<LibraryResource> = emptyList()
28+
override var codePreprocessors: List<CodePreprocessor> = emptyList()
29+
override var minKernelVersion: KotlinKernelVersion? = null
30+
override var originalDescriptorText: String? = null
31+
32+
companion object {
33+
internal fun build(buildAction: (LibraryDefinitionImpl) -> Unit): LibraryDefinition {
34+
return LibraryDefinitionImpl().also(buildAction)
35+
}
36+
}
37+
}
38+
39+
/**
40+
* Builds an instance of [LibraryDefinition].
41+
* Build action receives [LibraryDefinitionImpl] as an explicit argument
42+
* because of problems with names clashing that may arise.
43+
*/
44+
fun libraryDefinition(buildAction: (LibraryDefinitionImpl) -> Unit): LibraryDefinition {
45+
return LibraryDefinitionImpl.build(buildAction)
46+
}

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/libraries/LibraryDescriptor.kt

+13-13
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ import org.jetbrains.kotlinx.jupyter.api.ExactRendererTypeHandler
88
import org.jetbrains.kotlinx.jupyter.api.KotlinKernelVersion
99
import org.jetbrains.kotlinx.jupyter.api.libraries.CodeExecution
1010
import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinition
11-
import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinitionImpl
1211
import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryResource
12+
import org.jetbrains.kotlinx.jupyter.api.libraries.libraryDefinition
1313
import org.jetbrains.kotlinx.jupyter.config.currentKernelVersion
1414
import org.jetbrains.kotlinx.jupyter.exceptions.ReplPreprocessingException
1515
import org.jetbrains.kotlinx.jupyter.util.KotlinKernelVersionSerializer
@@ -78,18 +78,18 @@ class LibraryDescriptor(
7878
}
7979

8080
private fun processDescriptor(mapping: Map<String, String>): LibraryDefinition {
81-
return LibraryDefinitionImpl(
82-
dependencies = dependencies.replaceVariables(mapping),
83-
repositories = repositories.replaceVariables(mapping),
84-
imports = imports.replaceVariables(mapping),
85-
init = init.replaceVariables(mapping),
86-
shutdown = shutdown.replaceVariables(mapping),
87-
initCell = initCell.replaceVariables(mapping),
88-
renderers = renderers.replaceVariables(mapping),
89-
resources = resources.replaceVariables(mapping),
90-
minKernelVersion = minKernelVersion,
91-
originalDescriptorText = Json.encodeToString(this),
92-
)
81+
return libraryDefinition {
82+
it.dependencies = dependencies.replaceVariables(mapping)
83+
it.repositories = repositories.replaceVariables(mapping)
84+
it.imports = imports.replaceVariables(mapping)
85+
it.init = init.replaceVariables(mapping)
86+
it.shutdown = shutdown.replaceVariables(mapping)
87+
it.initCell = initCell.replaceVariables(mapping)
88+
it.renderers = renderers.replaceVariables(mapping)
89+
it.resources = resources.replaceVariables(mapping)
90+
it.minKernelVersion = minKernelVersion
91+
it.originalDescriptorText = Json.encodeToString(this)
92+
}
9393
}
9494

9595
companion object {

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/magics/CompoundCodePreprocessor.kt

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package org.jetbrains.kotlinx.jupyter.magics
2+
3+
import org.jetbrains.kotlinx.jupyter.api.CodePreprocessor
4+
import org.jetbrains.kotlinx.jupyter.api.KotlinKernelHost
5+
6+
/**
7+
* Containing [preprocessors]' [process] are run in reversed order: last added processors
8+
* are run first
9+
*/
10+
class CompoundCodePreprocessor(
11+
private val preprocessors: MutableList<CodePreprocessor>
12+
) : CodePreprocessor {
13+
constructor(vararg preprocessors: CodePreprocessor) : this(preprocessors.toMutableList())
14+
15+
override fun process(code: String, host: KotlinKernelHost): CodePreprocessor.Result {
16+
return preprocessors.foldRight(CodePreprocessor.Result(code, emptyList())) { preprocessor, result ->
17+
if (preprocessor.accepts(result.code)) {
18+
val newResult = preprocessor.process(result.code, host)
19+
CodePreprocessor.Result(newResult.code, result.libraries + newResult.libraries)
20+
} else result
21+
}
22+
}
23+
24+
fun add(preprocessor: CodePreprocessor) {
25+
preprocessors.add(preprocessor)
26+
}
27+
28+
fun addAll(preprocessors: Iterable<CodePreprocessor>) {
29+
this.preprocessors.addAll(preprocessors)
30+
}
31+
}

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/magics/MagicsProcessor.kt

+8-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package org.jetbrains.kotlinx.jupyter.magics
22

3-
import org.jetbrains.kotlinx.jupyter.api.Code
4-
import org.jetbrains.kotlinx.jupyter.api.libraries.LibraryDefinitionProducer
3+
import org.jetbrains.kotlinx.jupyter.api.CodePreprocessor
4+
import org.jetbrains.kotlinx.jupyter.api.KotlinKernelHost
55
import org.jetbrains.kotlinx.jupyter.common.ReplLineMagic
66
import org.jetbrains.kotlinx.jupyter.compiler.util.CodeInterval
77
import org.jetbrains.kotlinx.jupyter.exceptions.ReplPreprocessingException
@@ -10,8 +10,8 @@ import kotlin.script.experimental.jvm.util.determineSep
1010
class MagicsProcessor(
1111
private val handler: MagicsHandler,
1212
private val parseOutCellMarker: Boolean = false,
13-
) {
14-
fun processMagics(code: String, parseOnly: Boolean = false, tryIgnoreErrors: Boolean = false): MagicProcessingResult {
13+
) : CodePreprocessor {
14+
fun processMagics(code: String, parseOnly: Boolean = false, tryIgnoreErrors: Boolean = false): CodePreprocessor.Result {
1515
val magics = magicsIntervals(code)
1616

1717
for (magicRange in magics) {
@@ -38,7 +38,7 @@ class MagicsProcessor(
3838

3939
val codes = codeIntervals(code, magics, true)
4040
val preprocessedCode = codes.joinToString("") { code.substring(it.from, it.to) }
41-
return MagicProcessingResult(preprocessedCode, handler.getLibraries())
41+
return CodePreprocessor.Result(preprocessedCode, handler.getLibraries())
4242
}
4343

4444
fun codeIntervals(
@@ -79,7 +79,9 @@ class MagicsProcessor(
7979
}
8080
}
8181

82-
data class MagicProcessingResult(val code: Code, val libraries: List<LibraryDefinitionProducer>)
82+
override fun process(code: String, host: KotlinKernelHost): CodePreprocessor.Result {
83+
return processMagics(code)
84+
}
8385

8486
companion object {
8587
private const val MAGICS_SIGN = '%'

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt

+5-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import jupyter.kotlin.KotlinKernelHostProvider
77
import jupyter.kotlin.Repository
88
import org.jetbrains.kotlin.config.KotlinCompilerVersion
99
import org.jetbrains.kotlinx.jupyter.api.Code
10+
import org.jetbrains.kotlinx.jupyter.api.CodePreprocessor
1011
import org.jetbrains.kotlinx.jupyter.api.ExecutionCallback
1112
import org.jetbrains.kotlinx.jupyter.api.KotlinKernelHost
1213
import org.jetbrains.kotlinx.jupyter.api.KotlinKernelVersion
@@ -40,6 +41,7 @@ import org.jetbrains.kotlinx.jupyter.libraries.LibrariesScanner
4041
import org.jetbrains.kotlinx.jupyter.libraries.LibraryResourcesProcessorImpl
4142
import org.jetbrains.kotlinx.jupyter.libraries.ResolutionInfoProvider
4243
import org.jetbrains.kotlinx.jupyter.libraries.ResolutionInfoSwitcher
44+
import org.jetbrains.kotlinx.jupyter.magics.CompoundCodePreprocessor
4345
import org.jetbrains.kotlinx.jupyter.magics.FullMagicsHandler
4446
import org.jetbrains.kotlinx.jupyter.magics.MagicsProcessor
4547
import org.jetbrains.kotlinx.jupyter.repl.CellExecutor
@@ -231,6 +233,8 @@ class ReplForJupyterImpl(
231233
)
232234
)
233235

236+
private val codePreprocessor = CompoundCodePreprocessor(magics)
237+
234238
private val importsCollector: ScriptImportsCollector = ScriptImportsCollectorImpl()
235239

236240
// Used for various purposes, i.e. completion and listing errors
@@ -328,7 +332,7 @@ class ReplForJupyterImpl(
328332
fileAnnotationsProcessor,
329333
fieldsProcessor,
330334
typeRenderersProcessor,
331-
magics,
335+
codePreprocessor,
332336
resourcesProcessor,
333337
librariesScanner,
334338
notebook,

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl/impl/CellExecutorImpl.kt

+2-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ internal class CellExecutorImpl(private val replContext: SharedReplContext) : Ce
4545

4646
log.debug("Executing code:\n$code")
4747
val preprocessedCode = if (processMagics) {
48-
val processedMagics = magicsProcessor.processMagics(code)
48+
val processedMagics = codePreprocessor.process(code, context)
4949

5050
log.debug("Adding ${processedMagics.libraries.size} libraries")
5151
processedMagics.libraries.getDefinitions(notebook).forEach {
@@ -128,6 +128,7 @@ internal class CellExecutorImpl(private val replContext: SharedReplContext) : Ce
128128
library.classAnnotations.forEach(sharedContext.classAnnotationsProcessor::register)
129129
library.fileAnnotations.forEach(sharedContext.fileAnnotationsProcessor::register)
130130
sharedContext.afterCellExecution.addAll(library.afterCellExecution)
131+
sharedContext.codePreprocessor.addAll(library.codePreprocessors)
131132

132133
val classLoader = sharedContext.evaluator.lastClassLoader
133134
rethrowAsLibraryException(LibraryProblemPart.RESOURCES) {

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl/impl/SharedReplContext.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,15 @@ import org.jetbrains.kotlinx.jupyter.codegen.FileAnnotationsProcessor
99
import org.jetbrains.kotlinx.jupyter.codegen.ResultsTypeRenderersProcessor
1010
import org.jetbrains.kotlinx.jupyter.libraries.LibrariesScanner
1111
import org.jetbrains.kotlinx.jupyter.libraries.LibraryResourcesProcessor
12-
import org.jetbrains.kotlinx.jupyter.magics.MagicsProcessor
12+
import org.jetbrains.kotlinx.jupyter.magics.CompoundCodePreprocessor
1313
import org.jetbrains.kotlinx.jupyter.repl.InternalEvaluator
1414

1515
internal data class SharedReplContext(
1616
val classAnnotationsProcessor: ClassAnnotationsProcessor,
1717
val fileAnnotationsProcessor: FileAnnotationsProcessor,
1818
val fieldsProcessor: FieldsProcessor,
1919
val typeRenderersProcessor: ResultsTypeRenderersProcessor,
20-
val magicsProcessor: MagicsProcessor,
20+
val codePreprocessor: CompoundCodePreprocessor,
2121
val resourcesProcessor: LibraryResourcesProcessor,
2222
val librariesScanner: LibrariesScanner,
2323
val notebook: Notebook,

0 commit comments

Comments
 (0)