diff --git a/.gitignore b/.gitignore index 5e567f110..a22954663 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ artifacts/ **/.ipynb* **/build/ *.DS_Store -gradle.properties +/venv/ diff --git a/README.md b/README.md index 48ec7d2f9..d306dee99 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ # Kotlin kernel for IPython/Jupyter -[Kotlin](https://kotlinlang.org/) (1.3.70) kernel for [Jupyter](https://jupyter.org). +[Kotlin](https://kotlinlang.org/) (1.4.0) kernel for [Jupyter](https://jupyter.org). Alpha version. Tested with Jupyter 6.0.1 on OS X so far. diff --git a/build.gradle b/build.gradle deleted file mode 100644 index 0433efd04..000000000 --- a/build.gradle +++ /dev/null @@ -1,364 +0,0 @@ -import groovy.json.JsonOutput - -import java.nio.file.Path -import java.nio.file.Paths -import java.util.regex.Pattern -import java.util.stream.Collectors - -buildscript { - ext.shadowJarVersion = "5.2.0" - ext.kotlinVersion = '1.3.70-eap-3' - ext.baseVersion = '0.7.41' - repositories { - jcenter() - mavenLocal() - mavenCentral() - maven { url 'https://dl.bintray.com/kotlin/kotlin-eap' } - maven { url 'https://dl.bintray.com/kotlin/kotlin-dev' } - } - dependencies { - //noinspection DifferentKotlinGradleVersion - classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" - classpath "com.github.jengelman.gradle.plugins:shadow:$shadowJarVersion" - } -} - -allprojects { - apply plugin: 'kotlin' - - repositories { - jcenter() - mavenLocal() - mavenCentral() - // only when using Kotlin EAP releases ... - maven { url 'https://dl.bintray.com/kotlin/kotlin-eap' } - maven { url 'https://dl.bintray.com/kotlin/kotlin-dev' } - maven { url 'https://kotlin.bintray.com/kotlin-dependencies' } - } - - dependencies { - implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion" - - testImplementation 'junit:junit:4.12' - testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion" - } - - ext { - packageName = "kotlin-jupyter-kernel" - versionFileName = "VERSION" - - String artifactsPathStr = rootProject.findProperty('artifactsPath') ?: 'artifacts' - String installPath = rootProject.findProperty('installPath') - - //noinspection GroovyAssignabilityCheck - rootPath = rootDir.toPath() - //noinspection GroovyAssignabilityCheck - artifactsDir = rootPath.resolve(artifactsPathStr) - - //noinspection GroovyAssignabilityCheck - installPathLocal = installPath ? Paths.get(installPath) : - Paths.get(System.properties['user.home'].toString(), ".ipython", "kernels", "kotlin") - //noinspection GroovyAssignabilityCheck - distributionPath = rootPath.resolve("distrib") - //noinspection GroovyAssignabilityCheck - distribBuildPath = rootPath.resolve("distrib-build") - //noinspection GroovyAssignabilityCheck - logosPath = getSubDir(rootPath, "resources", "logos") - - if (project == rootProject) { - isProtectedBranch = isProtectedBranch() - String buildCounterStr = rootProject.findProperty('build.counter') ?: '100500' - String buildNumber = rootProject.findProperty('build.number') ?: '' - String devAddition = isProtectedBranch ? '' : '.dev1' - String defaultBuildNumber = "$baseVersion.$buildCounterStr$devAddition" - String buildNumberRegex = "[0-9]+(\\.[0-9]+){3}(\\.dev[0-9]+)?" - - if (!Pattern.matches(buildNumberRegex, buildNumber)) { - def versionFile = artifactsDir.resolve(versionFileName).toFile() - if (versionFile.exists()) { - def lines = versionFile.readLines() - assert !lines.empty, "There should be at least one line in VERSION file" - buildNumber = lines.first().trim() - } else { - buildNumber = defaultBuildNumber - } - } - - project.version = buildNumber - println("##teamcity[buildNumber '$version']") - } else { - isProtectedBranch = rootProject.isProtectedBranch - project.version = rootProject.version - } - - debugPort = 1044 - debuggerConfig = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$debugPort".toString() - - mainSourceSetDir = "main" - resourcesDir = "resources" - runtimePropertiesFile = "runtime.properties" - - jarsPath = "jars" - librariesPath = "libraries" - kernelFile = "kernel.json" - configDir = "config" - // Straight slash is used 'cause it's universal across the platforms, and is used in jar_args config - jarArgsFile = configDir + "/jar_args.json" - - distribKernelDir = "kernel" - runKernelDir = "run_kotlin_kernel" - setupPy = "setup.py" - runKernelPy = "run_kernel.py" - - condaMetaDir = "conda" - condaMetaYaml = "meta.yaml" - condaPackageDir = "conda-package" - condaPackageName = packageName - condaPackageFileName = "$condaPackageName-${version}-py_0.tar.bz2".toString() - - pyPiPackageDir = "pip-package" - pyPiPackageName = packageName.replaceAll("-", "_") - pyPiPackageFileName = "$pyPiPackageName-${version}-py3-none-any.whl".toString() - - copyLibrariesTaskPrefix = "copyLibraries" - installLibsTaskPrefix = "installLibs" - installKernelTaskPrefix = "installKernel" - cleanInstallDirTaskPrefix = "cleanInstallDir" - - localGroup = "local install" - distribGroup = "distrib" - condaGroup = "conda" - pyPiGroup = "pip" - buildGroup = "build" - - condaUserStable = rootProject.findProperty('condaUserStable') ?: '' - condaPasswordStable = rootProject.findProperty('condaPasswordStable') ?: '' - condaUserDev = rootProject.findProperty('condaUserDev') ?: '' - - stablePyPiUser = rootProject.findProperty('stablePyPiUser') ?: '' - stablePyPiPassword = rootProject.findProperty('stablePyPiPassword') ?: '' - devPyPiUser = rootProject.findProperty('devPyPiUser') ?: '' - devPyPiPassword = rootProject.findProperty('devPyPiPassword') ?: '' - } -} - -apply plugin: 'com.github.johnrengelman.shadow' - -configurations { - deploy -} - -dependencies { - compile project(":jupyter-lib") - compile "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-scripting-jvm-host-embeddable:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-scripting-common:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-compiler-embeddable:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-script-util:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-main-kts:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-scripting-dependencies:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-scripting-dependencies-maven:$kotlinVersion" - - compile "org.apache.maven:maven-core:3.0.3" - compile 'org.slf4j:slf4j-api:1.7.25' - compile 'khttp:khttp:1.0.0' - compile 'org.zeromq:jeromq:0.3.5' - compile 'com.beust:klaxon:5.2' - compile 'com.github.ajalt:clikt:2.3.0' - runtime 'org.slf4j:slf4j-simple:1.7.25' - runtime "org.jetbrains.kotlin:jcabi-aether:1.0-dev-3" - runtime "org.sonatype.aether:aether-api:1.13.1" - runtime "net.java.dev.jna:jna:5.4.0" - - deploy project(":jupyter-lib") -} - -jar.manifest.attributes( - 'Main-class': 'org.jetbrains.kotlin.jupyter.IkotlinKt', - 'Implementation-Version': version -) - -task buildProperties(group: buildGroup) { - def outputDir = file(getSubDir(buildDir.toPath(), resourcesDir, mainSourceSetDir)) - - inputs.property "version", version - outputs.dir outputDir - - doLast { - outputDir.mkdirs() - def propertiesFile = file(getSubDir(outputDir.toPath(), runtimePropertiesFile)) - propertiesFile.text = inputs.properties.entrySet().stream().map { - "${it.key}=${it.value}\n" - }.collect(Collectors.joining()) - } -} - -processResources { - dependsOn buildProperties -} - -shadowJar { - archiveBaseName.set(packageName) - archiveClassifier.set('') - mergeServiceFiles() -} - -boolean isProtectedBranch() { - def branchProp = 'build.branch' - def branch = project.findProperty(branchProp) as String - println("Current branch: $branch") - if (branch != null) { - branch = branch.substring(branch.lastIndexOf('/') + 1) - return branch == 'master' - } - return false -} - -static String makeTaskName(String prefix, Boolean local) { - return prefix + (local ? "Local" : "Distrib") -} - -static void makeDirs(Path path) { - File dir = path.toFile() - if (!dir.exists()) { - dir.mkdirs() - } -} - -static Path getSubDir(Path dir, String... subDir) { - def newDir = dir - for (s in subDir) { - newDir = newDir.resolve(s) - } - return newDir -} - -static void writeJson(Map json, Path path) { - def str = JsonOutput.prettyPrint(JsonOutput.toJson(json)) - path.toFile().write(str, 'UTF-8') -} - -void createCleanTasks() { - [true, false].forEach { local -> - def dir = local ? installPathLocal : distribBuildPath - task(makeTaskName(cleanInstallDirTaskPrefix, local)) { - group(local ? localGroup : distribGroup) - doLast { - dir.deleteDir() - } - } - } -} -createCleanTasks() - -@SuppressWarnings("unused") -void createInstallTasks(Boolean local, Path specPath, Path mainInstallPath) { - def groupName = local ? localGroup : distribGroup - def cleanDirTask = getTasks().getByName(makeTaskName(cleanInstallDirTaskPrefix, local)) - def args = [type: Copy, dependsOn: cleanDirTask, group: groupName] - - task (args, makeTaskName(copyLibrariesTaskPrefix, local)) { - from librariesPath - into mainInstallPath.resolve(librariesPath) - } - - task (args, makeTaskName(installLibsTaskPrefix, local)) { - from configurations.deploy - into mainInstallPath.resolve(jarsPath) - } - - task ([type: Copy, group: groupName, dependsOn: [cleanDirTask, shadowJar]], makeTaskName(installKernelTaskPrefix, local)) { - from shadowJar.outputs - into mainInstallPath.resolve(jarsPath) - } - - [true, false].forEach { debug -> - def specTaskName = createTaskForSpecs(debug, local, groupName, cleanDirTask, specPath, mainInstallPath) - createMainInstallTask(debug, local, groupName, specTaskName) - } -} - -String createTaskForSpecs(Boolean debug, Boolean local, String group, Task cleanDir, Path specPath, Path mainInstallPath) { - String taskName = makeTaskName(debug ? "createDebugSpecs" : "createSpecs", local) - task([group: group], taskName) { - dependsOn cleanDir, shadowJar - doLast { - File kernelFile = files { shadowJar }.singleFile - - List libsCp = files { configurations.deploy }.files.collect { - it.name - }.asList() - - makeDirs(mainInstallPath.resolve(jarsPath)) - makeDirs(mainInstallPath.resolve(configDir)) - makeDirs(specPath) - - makeJarArgs(mainInstallPath, kernelFile.name, libsCp, debug ? debuggerConfig : "") - makeKernelSpec(specPath, local) - } - } - return taskName -} - -void createMainInstallTask(Boolean debug, Boolean local, String group, String specsTaskName) { - def taskNamePrefix = local ? "install" : "prepare" - def taskNameMiddle = debug ? "Debug": "" - def taskNameSuffix = local ? "" : "Package" - def taskName = "$taskNamePrefix$taskNameMiddle$taskNameSuffix" - - def dependencies = [ - local ? copyRunKernelPy : prepareDistributionDir, - makeTaskName(installKernelTaskPrefix, local), - makeTaskName(installLibsTaskPrefix, local), - specsTaskName, - makeTaskName(copyLibrariesTaskPrefix, local) - ] - - task([group: group], taskName) { - dependsOn(dependencies) - } -} - -void makeKernelSpec(Path installPath, Boolean localInstall) { - def argv = localInstall ? - Arrays.asList("python", - installPath.resolve(runKernelPy).toString(), - "{connection_file}", - installPath.resolve(jarArgsFile).toString(), - installPath.toString()) : - Arrays.asList("python", "-m", "run_kotlin_kernel", "{connection_file}") - - writeJson([ - "display_name": "Kotlin", - "language": "kotlin", - "argv": argv, - ], installPath.resolve(kernelFile)) - - project.copy { - from logosPath - into installPath - } -} - -void makeJarArgs(Path installPath, String kernelJarPath, List classPath, String debuggerConfig = "") { - writeJson([ - "mainJar": kernelJarPath, - "classPath": classPath, - "debuggerConfig": debuggerConfig, - ], installPath.resolve(project.jarArgsFile)) -} - -/****** Local install ******/ - -task copyRunKernelPy(type: Copy, dependsOn: cleanInstallDirLocal, group: localGroup) { - from distributionPath.resolve(runKernelDir).resolve(runKernelPy) - into installPathLocal -} - -createInstallTasks(true, installPathLocal, installPathLocal) - -task uninstall(dependsOn: cleanInstallDirLocal, group: localGroup) - -apply from: 'distrib.gradle' diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 000000000..c7456123c --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,247 @@ +@file:Suppress("UnstableApiUsage") + +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import java.nio.file.Path +import java.nio.file.Paths + +val packageName by extra("kotlin-jupyter-kernel") +val baseVersion: String by project + +plugins { + kotlin("jvm") + id("com.github.johnrengelman.shadow") +} + +class TaskOptions: AllOptions { + override val versionFileName = "VERSION" + override val rootPath: Path = rootDir.toPath() + + override val isLocalBuild = (rootProject.findProperty("build.isLocal") ?: "false") == "true" + + override val artifactsDir: Path = { + val artifactsPathStr = rootProject.findProperty("artifactsPath") as? String ?: "artifacts" + val artifactsDir = rootPath.resolve(artifactsPathStr) + + if (isLocalBuild) + project.delete(artifactsDir) + + project.version = detectVersion(baseVersion, artifactsDir, versionFileName) + println("##teamcity[buildNumber '$version']") + artifactsDir + }() + + private val installPath = rootProject.findProperty("installPath") as String? + + override val librariesPath = "libraries" + override val installPathLocal: Path = if (installPath != null) + Paths.get(installPath) + else + Paths.get(System.getProperty("user.home").toString(), ".ipython", "kernels", "kotlin") + + override val resourcesDir = "resources" + override val distribBuildPath: Path = rootPath.resolve("distrib-build") + override val logosPath = getSubDir(rootPath, resourcesDir, "logos") + override val nbExtensionPath = getSubDir(rootPath, resourcesDir, "notebook-extension") + override val distributionPath: Path by extra(rootPath.resolve("distrib")) + override val jarsPath = "jars" + override val configDir = "config" + + // Straight slash is used 'cause it's universal across the platforms, and is used in jar_args config + override val jarArgsFile = "$configDir/jar_args.json" + override val runKernelPy = "run_kernel.py" + override val kernelFile = "kernel.json" + override val installKernelTaskPrefix = "installKernel" + override val cleanInstallDirTaskPrefix = "cleanInstallDir" + override val copyLibrariesTaskPrefix = "copyLibraries" + override val installLibsTaskPrefix = "installLibs" + + override val localGroup = "local install" + override val distribGroup = "distrib" + override val condaGroup = "conda" + override val pyPiGroup = "pip" + override val buildGroup = "build" + + private val debugPort = 1044 + override val debuggerConfig = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$debugPort" + + override val mainSourceSetDir = "main" + override val runtimePropertiesFile = "runtime.properties" + + override val distribKernelDir = "kernel" + override val runKernelDir = "run_kotlin_kernel" + override val setupPy = "setup.py" + + override val copyRunKernelPy: Task + get() = tasks.getByName("copyRunKernelPy") + override val prepareDistributionDir: Task + get() = tasks.getByName("prepareDistributionDir") + override val cleanInstallDirDistrib: Task + get() = tasks.getByName("cleanInstallDirDistrib") + + override val isOnProtectedBranch: Boolean + get() = extra["isOnProtectedBranch"] as Boolean + + override val distribUtilsPath: Path = rootPath.resolve("distrib-util") + override val distribUtilRequirementsPath: Path = distribUtilsPath.resolve("requirements-common.txt") + override val distribUtilRequirementsHintsRemPath: Path = distribUtilsPath.resolve("requirements-hints-remover.txt") + override val removeTypeHints = true + override val typeHintsRemover: Path = distribUtilsPath.resolve("remove_type_hints.py") + + override val condaTaskSpecs = { + val condaUserStable = stringPropOrEmpty("condaUserStable") + val condaPasswordStable = stringPropOrEmpty("condaPasswordStable") + val condaUserDev = stringPropOrEmpty("condaUserDev") + + val condaPackageSettings = object : DistributionPackageSettings { + override val dir = "conda-package" + override val name = packageName + override val fileName by lazy {"$name-${version}-py_0.tar.bz2"} + } + + val condaCredentials = CondaCredentials(condaUserStable, condaPasswordStable) + UploadTaskSpecs( + condaPackageSettings,"conda", condaGroup, + CondaTaskSpec( + condaUserStable, + condaCredentials + ), + CondaTaskSpec( + condaUserDev, + condaCredentials + ) + ) + }() + + override val pyPiTaskSpecs = { + val stablePyPiUser = stringPropOrEmpty("stablePyPiUser") + val stablePyPiPassword = stringPropOrEmpty("stablePyPiPassword") + val devPyPiUser = stringPropOrEmpty("devPyPiUser") + val devPyPiPassword = stringPropOrEmpty("devPyPiPassword") + + val pyPiPackageSettings = object : DistributionPackageSettings { + override val dir = "pip-package" + override val name = packageName.replace("-", "_") + override val fileName by lazy {"$name-${version}-py3-none-any.whl"} + } + + UploadTaskSpecs( + pyPiPackageSettings, "pyPi", pyPiGroup, + PyPiTaskSpec( + "https://upload.pypi.org/legacy/", + stablePyPiUser, + stablePyPiPassword + ), + PyPiTaskSpec( + "https://test.pypi.org/legacy/", + devPyPiUser, + devPyPiPassword + ) + ) + }() + +} + +allprojects { + val kotlinLanguageLevel: String by rootProject + val jvmTarget: String by rootProject + + tasks.withType(KotlinCompile::class.java).all { + kotlinOptions { + languageVersion = kotlinLanguageLevel + this.jvmTarget = jvmTarget + } + } + + repositories { + jcenter() + mavenLocal() + mavenCentral() + // only when using Kotlin EAP releases ... + maven { url = uri("https://dl.bintray.com/kotlin/kotlin-eap") } + maven { url = uri("https://dl.bintray.com/kotlin/kotlin-dev") } + maven { url = uri("https://kotlin.bintray.com/kotlin-dependencies") } + } +} + +val deploy: Configuration by configurations.creating + +dependencies { + implementation(kotlinDep("kotlin-stdlib")) + testImplementation("junit:junit:4.12") + testImplementation(kotlinDep("kotlin-test")) + + implementation(project(":jupyter-lib")) + implementation(kotlinDep("kotlin-reflect")) + implementation(kotlinDep("kotlin-scripting-ide-services-embeddable")) { isTransitive = false } + implementation(kotlinDep("kotlin-scripting-common")) + implementation(kotlinDep("kotlin-scripting-compiler-embeddable")) + implementation(kotlinDep("kotlin-compiler-embeddable")) + implementation(kotlinDep("kotlin-stdlib-jdk8")) + implementation(kotlinDep("kotlin-script-util")) + implementation(kotlinDep("kotlin-scripting-dependencies")) + implementation(kotlinDep("kotlin-scripting-dependencies-maven")) + implementation(kotlinDep("kotlin-main-kts")) + + implementation("org.apache.maven:maven-core:3.0.3") + implementation("org.slf4j:slf4j-api:1.7.25") + implementation("khttp:khttp:1.0.0") + implementation("org.zeromq:jeromq:0.3.5") + implementation("com.beust:klaxon:5.2") + implementation("com.github.ajalt:clikt:2.3.0") + runtimeOnly("org.slf4j:slf4j-simple:1.7.25") + runtimeOnly("org.jetbrains.kotlin:jcabi-aether:1.0-dev-3") + runtimeOnly("org.sonatype.aether:aether-api:1.13.1") + runtimeOnly("net.java.dev.jna:jna:5.4.0") + + deploy(project(":jupyter-lib")) +} + +val jar by tasks.getting(Jar::class) { + manifest { + attributes["Main-Class"] = "org.jetbrains.kotlin.jupyter.IkotlinKt" + attributes["Implementation-Version"] = project.version + } +} + +with(ProjectWithOptionsImpl(project, TaskOptions())) { + /****** Build tasks ******/ + tasks.withType { + archiveBaseName.set(packageName) + archiveClassifier.set("") + mergeServiceFiles() + + manifest { + attributes["Main-Class"] = "org.jetbrains.kotlin.jupyter.IkotlinKt" + } + } + + task("buildProperties") { + group = buildGroup + val outputDir = file(getSubDir(buildDir.toPath(), resourcesDir, mainSourceSetDir)) + + inputs.property("version", version) + outputs.dir(outputDir) + + doLast { + outputDir.mkdirs() + val propertiesFile = file(getSubDir(outputDir.toPath(), runtimePropertiesFile)) + propertiesFile.writeText(inputs.properties.entries.joinToString("") { "${it.key}=${it.value}\n" }) + } + } + + tasks.processResources { + dependsOn("buildProperties") + } + + createCleanTasks() + + /****** Local install ******/ + prepareLocalTasks() + + /****** Distribution ******/ + prepareDistributionTasks() + createInstallTasks(false, distribBuildPath.resolve(distribKernelDir), distribBuildPath.resolve(runKernelDir)) + prepareCondaTasks() + preparePyPiTasks() +} diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 000000000..dabaed904 --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,9 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + `kotlin-dsl` +} + +repositories { + jcenter() +} diff --git a/buildSrc/src/main/kotlin/distTasks.kt b/buildSrc/src/main/kotlin/distTasks.kt new file mode 100644 index 000000000..a7690b59e --- /dev/null +++ b/buildSrc/src/main/kotlin/distTasks.kt @@ -0,0 +1,151 @@ +import org.gradle.api.tasks.Copy +import org.gradle.api.tasks.Exec +import org.gradle.kotlin.dsl.register +import java.io.ByteArrayInputStream +import java.nio.file.Path + +private fun ProjectWithDistribOptions.removeTypeHintsIfNeeded(files: List) { + if (!removeTypeHints) + return + + files.forEach { + val fileName = it.toAbsolutePath().toString() + exec { + commandLine("python", typeHintsRemover, fileName, fileName) + } + } +} + +fun ProjectWithOptions.prepareDistributionTasks() { + tasks.register("installCommonRequirements") { + group = distribGroup + requirementsFile = distribUtilRequirementsPath + } + + tasks.register("installHintRemoverRequirements") { + group = distribGroup + requirementsFile = distribUtilRequirementsHintsRemPath + } + + tasks.register("copyDistribFiles") { + group = distribGroup + dependsOn("cleanInstallDirDistrib") + if(removeTypeHints) { + dependsOn("installHintRemoverRequirements") + } + from(distributionPath) + into(distribBuildPath) + exclude(".idea/**") + + val pythonFiles = mutableListOf() + eachFile { + val absPath = distribBuildPath.resolve(this.path).toAbsolutePath() + if(this.path.endsWith(".py")) + pythonFiles.add(absPath) + } + + doLast { + removeTypeHintsIfNeeded(pythonFiles) + } + } + + task("prepareDistributionDir") { + group = distribGroup + dependsOn("cleanInstallDirDistrib", "copyDistribFiles") + doLast { + val versionFilePath = distribBuildPath.resolve(versionFileName) + versionFilePath.toFile().writeText(version as String) + project.copy { + from(versionFilePath) + into(artifactsDir) + } + } + } +} + +fun ProjectWithOptions.prepareCondaTasks() { + with (condaTaskSpecs) { + tasks.register("condaPackage") { + group = condaGroup + dependsOn("cleanInstallDirDistrib", "preparePackage") + commandLine("conda-build", "conda", "--output-folder", packageSettings.dir) + workingDir(distribBuildPath) + doLast { + copy { + from(distribBuildPath.resolve(packageSettings.dir).resolve("noarch").resolve(packageSettings.fileName)) + into(artifactsDir) + } + } + } + + condaTaskSpecs.createTasks(this@prepareCondaTasks) { taskSpec -> + task(taskSpec.taskName) { + group = condaGroup + val artifactPath = artifactsDir.resolve(packageSettings.fileName) + + if (!artifactPath.toFile().exists()) { + dependsOn("cleanInstallDirDistrib", "condaPackage") + } + + doLast { + exec { + commandLine ("anaconda", "login", + "--username", taskSpec.credentials.username, + "--password", taskSpec.credentials.password) + + standardInput = ByteArrayInputStream("yes".toByteArray()) + } + + exec { + commandLine("anaconda", "upload", "-u", taskSpec.username, artifactPath.toString()) + } + } + } + } + } +} + +fun ProjectWithOptions.preparePyPiTasks() { + with(pyPiTaskSpecs) { + tasks.register("pyPiPackage") { + group = pyPiGroup + + dependsOn("preparePackage") + if (isLocalBuild) { + dependsOn("installCommonRequirements") + } + + commandLine("python", setupPy, "bdist_wheel", + "--dist-dir", packageSettings.dir) + workingDir(distribBuildPath) + + doLast { + copy { + from(distribBuildPath.resolve(packageSettings.dir).resolve(packageSettings.fileName)) + into(artifactsDir) + } + } + } + + pyPiTaskSpecs.createTasks(this@preparePyPiTasks) { taskSpec -> + tasks.register(taskSpec.taskName) { + group = pyPiGroup + workingDir(artifactsDir) + val artifactPath = artifactsDir.resolve(packageSettings.fileName) + + if (isLocalBuild) { + dependsOn("installCommonRequirements") + } + if (!artifactPath.toFile().exists()) { + dependsOn("pyPiPackage") + } + + commandLine("twine", "upload", + "-u", taskSpec.username, + "-p", taskSpec.password, + "--repository-url", taskSpec.repoURL, + packageSettings.fileName) + } + } + } +} diff --git a/buildSrc/src/main/kotlin/distribution.kt b/buildSrc/src/main/kotlin/distribution.kt new file mode 100644 index 000000000..d5c57b1ad --- /dev/null +++ b/buildSrc/src/main/kotlin/distribution.kt @@ -0,0 +1,94 @@ +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.tasks.Exec +import org.gradle.api.tasks.InputFile +import java.nio.file.Path + +interface DistribOptions { + val isOnProtectedBranch: Boolean + val cleanInstallDirDistrib: Task + + val condaGroup: String + val pyPiGroup: String + val buildGroup: String + + val condaTaskSpecs: UploadTaskSpecs + val pyPiTaskSpecs: UploadTaskSpecs + + val distribKernelDir: String + val distributionPath: Path + + val distribUtilsPath: Path + val distribUtilRequirementsPath: Path + val distribUtilRequirementsHintsRemPath: Path + val removeTypeHints: Boolean + val typeHintsRemover: Path +} + +interface ProjectWithDistribOptions: Project, DistribOptions + +open class TaskSpec ( + var taskName: String = "" +) + +interface DistributionPackageSettings { + val dir: String + val name: String + val fileName: String +} + +class UploadTaskSpecs ( + val packageSettings: DistributionPackageSettings, + private val repoName: String, + private val taskGroup: String, + private val stable: T, private val dev: T +) { + init { + this.stable.taskName = taskName("Stable") + this.dev.taskName = taskName("Dev") + } + + private fun taskName(type: String) = repoName + "Upload" + type + + fun createTasks(project: ProjectWithDistribOptions, taskCreationAction: (T) -> Unit) { + with(project) { + if (isOnProtectedBranch) { + taskCreationAction(stable) + } + taskCreationAction(dev) + + project.task(taskName("Protected")) { + dependsOn(cleanInstallDirDistrib) + group = taskGroup + if (isOnProtectedBranch) { + dependsOn(dev.taskName) + } + } + } + } +} + +class CondaCredentials ( + val username: String, + val password: String +) + +class CondaTaskSpec( + val username: String, + val credentials: CondaCredentials +) : TaskSpec() + +class PyPiTaskSpec ( + val repoURL: String, + val username: String, + val password: String +) : TaskSpec() + +abstract class PipInstallReq : Exec() { + @get:InputFile + var requirementsFile: Path? = null + set(value) { + commandLine("pip", "install", "-r", value) + field = value + } +} diff --git a/buildSrc/src/main/kotlin/install.kt b/buildSrc/src/main/kotlin/install.kt new file mode 100644 index 000000000..f2bca1135 --- /dev/null +++ b/buildSrc/src/main/kotlin/install.kt @@ -0,0 +1,176 @@ +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.tasks.Copy +import org.gradle.kotlin.dsl.get +import org.gradle.kotlin.dsl.register +import java.nio.file.Path + +interface BuildOptions { + val isLocalBuild: Boolean + val mainSourceSetDir: String + val runtimePropertiesFile: String + val resourcesDir: String + val versionFileName: String + val rootPath: Path + + val artifactsDir: Path +} + +interface ProjectWithBuildOptions: Project, BuildOptions + +interface InstallOptions { + val installPathLocal: Path + val distribBuildPath: Path + val logosPath: Path + val nbExtensionPath: Path + + val librariesPath: String + val jarsPath: String + val configDir: String + val jarArgsFile: String + val runKernelDir: String + val runKernelPy: String + val setupPy: String + val kernelFile: String + + val installKernelTaskPrefix: String + val cleanInstallDirTaskPrefix: String + val copyLibrariesTaskPrefix: String + val installLibsTaskPrefix: String + + val localGroup: String + val distribGroup: String + + val debuggerConfig: String + + val copyRunKernelPy: Task + val prepareDistributionDir: Task +} + +interface ProjectWithInstallOptions: Project, InstallOptions + +fun ProjectWithInstallOptions.createCleanTasks() { + listOf(true, false).forEach { local -> + val dir = if(local) installPathLocal else distribBuildPath + task(makeTaskName(cleanInstallDirTaskPrefix, local)) { + group = if(local) localGroup else distribGroup + doLast { + if(!dir.deleteDir()) { + throw Exception("Cannot delete $dir") + } + } + } + } +} + + +fun ProjectWithInstallOptions.createInstallTasks(local: Boolean, specPath: Path, mainInstallPath: Path) { + val groupName = if(local) localGroup else distribGroup + val cleanDirTask = tasks.getByName(makeTaskName(cleanInstallDirTaskPrefix, local)) + val shadowJar = tasks.getByName("shadowJar") + + tasks.register(makeTaskName(copyLibrariesTaskPrefix, local)) { + dependsOn(cleanDirTask) + group = groupName + from(librariesPath) + into(mainInstallPath.resolve(librariesPath)) + } + + tasks.register(makeTaskName(installLibsTaskPrefix, local)) { + dependsOn(cleanDirTask) + group = groupName + from(configurations["deploy"]) + into(mainInstallPath.resolve(jarsPath)) + } + + tasks.register(makeTaskName(installKernelTaskPrefix, local)) { + dependsOn(cleanDirTask, shadowJar) + group = groupName + from (shadowJar.outputs) + into (mainInstallPath.resolve(jarsPath)) + } + + listOf(true, false).forEach { debug -> + val specTaskName = createTaskForSpecs(debug, local, groupName, cleanDirTask, shadowJar, specPath, mainInstallPath) + createMainInstallTask(debug, local, groupName, specTaskName) + } +} + + +fun ProjectWithInstallOptions.createTaskForSpecs(debug: Boolean, local: Boolean, group: String, cleanDir: Task, shadowJar: Task, specPath: Path, mainInstallPath: Path): String { + val taskName = makeTaskName(if(debug) "createDebugSpecs" else "createSpecs", local) + tasks.register(taskName) { + this.group = group + dependsOn (cleanDir, shadowJar) + doLast { + val kernelFile = files(shadowJar).singleFile + + val libsCp = files(configurations["deploy"]).files.map { it.name } + + makeDirs(mainInstallPath.resolve(jarsPath)) + makeDirs(mainInstallPath.resolve(configDir)) + makeDirs(specPath) + + makeJarArgs(mainInstallPath, kernelFile.name, libsCp, if (debug) debuggerConfig else "") + makeKernelSpec(specPath, local) + } + } + return taskName +} + +fun ProjectWithInstallOptions.createMainInstallTask(debug: Boolean, local: Boolean, group: String, specsTaskName: String) { + val taskNamePrefix = if(local) "install" else "prepare" + val taskNameMiddle = if(debug) "Debug" else "" + val taskNameSuffix = if(local) "" else "Package" + val taskName = "$taskNamePrefix$taskNameMiddle$taskNameSuffix" + + val dependencies = listOf( + makeTaskName(cleanInstallDirTaskPrefix, local), + if(local) copyRunKernelPy else prepareDistributionDir, + makeTaskName(installKernelTaskPrefix, local), + makeTaskName(installLibsTaskPrefix, local), + specsTaskName, + makeTaskName(copyLibrariesTaskPrefix, local) + ) + + task(taskName) { + this.group = group + dependsOn(dependencies) + } +} + +fun ProjectWithInstallOptions.makeKernelSpec(installPath: Path, localInstall: Boolean) { + val argv = if(localInstall) { + listOf("python", + installPath.resolve(runKernelPy).toString(), + "{connection_file}", + installPath.resolve(jarArgsFile).toString(), + installPath.toString()) + } else { + listOf("python", "-m", "run_kotlin_kernel", "{connection_file}") + } + + writeJson(mapOf( + "display_name" to "Kotlin", + "language" to "kotlin", + "interrupt_mode" to "message", + "argv" to argv + ), installPath.resolve(kernelFile)) + + project.copy { + from (logosPath) + into (installPath) + } + project.copy { + from(nbExtensionPath) + into(installPath) + } +} + +fun ProjectWithInstallOptions.makeJarArgs(installPath: Path, kernelJarPath: String, classPath: List, debuggerConfig: String = "") { + writeJson(mapOf( + "mainJar" to kernelJarPath, + "classPath" to classPath, + "debuggerConfig" to debuggerConfig + ), installPath.resolve(jarArgsFile)) +} diff --git a/buildSrc/src/main/kotlin/localInstallTasks.kt b/buildSrc/src/main/kotlin/localInstallTasks.kt new file mode 100644 index 000000000..013b2e3c0 --- /dev/null +++ b/buildSrc/src/main/kotlin/localInstallTasks.kt @@ -0,0 +1,24 @@ +import org.gradle.api.tasks.Copy +import org.gradle.kotlin.dsl.register + +fun ProjectWithOptions.prepareLocalTasks() { + tasks.register("copyRunKernelPy") { + group = localGroup + dependsOn("cleanInstallDirLocal") + from(distributionPath.resolve(runKernelDir).resolve(runKernelPy)) + into(installPathLocal) + } + + tasks.register("copyNbExtension") { + group = localGroup + from(nbExtensionPath) + into(installPathLocal) + } + + createInstallTasks(true, installPathLocal, installPathLocal) + + task("uninstall") { + group = localGroup + dependsOn("cleanInstallDirLocal") + } +} diff --git a/buildSrc/src/main/kotlin/util.kt b/buildSrc/src/main/kotlin/util.kt new file mode 100644 index 000000000..a9f46a1fe --- /dev/null +++ b/buildSrc/src/main/kotlin/util.kt @@ -0,0 +1,35 @@ +import groovy.json.JsonOutput +import org.gradle.api.Project +import org.gradle.kotlin.dsl.provideDelegate +import java.nio.file.Path + +fun makeTaskName(prefix: String, local: Boolean) = prefix + (if (local) "Local" else "Distrib") + +fun makeDirs(path: Path) { + val dir = path.toFile() + if (!dir.exists()) { + dir.mkdirs() + } +} + +fun getSubDir(dir: Path, vararg subDir: String): Path = subDir.fold(dir, Path::resolve) + +fun writeJson(json: Map, path: Path) { + val str = JsonOutput.prettyPrint(JsonOutput.toJson(json)) + path.toFile().writeText(str, Charsets.UTF_8) +} + +fun Project.kotlinDep(dependency: String): String { + val kotlinVersion: String by project + return "org.jetbrains.kotlin:$dependency:$kotlinVersion" +} + +fun Path.deleteDir() = toFile().deleteRecursively() + +fun Project.stringPropOrEmpty(name: String) = rootProject.findProperty(name) as String? ?: "" + +interface AllOptions: BuildOptions, InstallOptions, DistribOptions +interface ProjectWithOptions : ProjectWithBuildOptions, ProjectWithInstallOptions, ProjectWithDistribOptions + +class ProjectWithOptionsImpl(private val p: Project, private val opt: AllOptions): + Project by p, InstallOptions by opt, DistribOptions by opt, BuildOptions by opt, ProjectWithOptions diff --git a/buildSrc/src/main/kotlin/versionDetection.kt b/buildSrc/src/main/kotlin/versionDetection.kt new file mode 100644 index 000000000..6df7798cb --- /dev/null +++ b/buildSrc/src/main/kotlin/versionDetection.kt @@ -0,0 +1,39 @@ +import org.gradle.api.Project +import org.gradle.kotlin.dsl.extra +import org.gradle.kotlin.dsl.invoke +import java.nio.file.Path +import java.util.regex.Pattern + +fun Project.isProtectedBranch(): Boolean { + val branchProp = "build.branch" + var branch = project.findProperty(branchProp) as String? + println("Current branch: $branch") + if (branch != null) { + branch = branch.substring(branch.lastIndexOf("/") + 1) + return branch == "master" + } + return false +} + +fun Project.detectVersion(baseVersion: String, artifactsDir: Path, versionFileName: String): String { + val isOnProtectedBranch by extra(isProtectedBranch()) + val buildCounterStr = rootProject.findProperty("build.counter") as String? ?: "100500" + val buildNumber = rootProject.findProperty("build.number") as String? ?: "" + val devCounter = rootProject.findProperty("build.devCounter") as String? ?: "1" + val devAddition = if(isOnProtectedBranch) "" else ".dev$devCounter" + val defaultBuildNumber = "$baseVersion.$buildCounterStr$devAddition" + val buildNumberRegex = "[0-9]+(\\.[0-9]+){3}(\\.dev[0-9]+)?" + + return if (!Pattern.matches(buildNumberRegex, buildNumber)) { + val versionFile = artifactsDir.resolve(versionFileName).toFile() + if (versionFile.exists()) { + val lines = versionFile.readLines() + assert(lines.isNotEmpty()) { "There should be at least one line in VERSION file" } + lines.first().trim() + } else { + defaultBuildNumber + } + } else { + buildNumber + } +} diff --git a/distrib-util/remove_type_hints.py b/distrib-util/remove_type_hints.py new file mode 100644 index 000000000..746b91d14 --- /dev/null +++ b/distrib-util/remove_type_hints.py @@ -0,0 +1,51 @@ +import ast +import astor +import sys + + +class TypeHintRemover(ast.NodeTransformer): + + def visit_FunctionDef(self, node): + # remove the return type definition + node.returns = None + # remove all argument annotations + if node.args.args: + for arg in node.args.args: + arg.annotation = None + self.generic_visit(node) + return node + + def visit_AnnAssign(self, node): + if node.value is None: + return None + return ast.Assign([node.target], node.value) + + def visit_Import(self, node): + node.names = [n for n in node.names if n.name != 'typing'] + return node if node.names else None + + def visit_ImportFrom(self, node): + return node if node.module != 'typing' else None + + +def remove_type_hints(source: str): + # parse the source code into an AST + parsed_source = ast.parse(source) + # remove all type annotations, function return type definitions + # and import statements from 'typing' + transformed = TypeHintRemover().visit(parsed_source) + # convert the AST back to source code + return astor.to_source(transformed) + + +def main(): + _, source_name, dest_name = sys.argv + with open(source_name, "r") as sourceFile: + source = "\n".join(sourceFile.readlines()) + dest = remove_type_hints(source) + with open(dest_name, "w") as destFile: + destFile.write(dest) + + +if __name__ == "__main__": + main() diff --git a/distrib-util/requirements-common.txt b/distrib-util/requirements-common.txt new file mode 100644 index 000000000..8ca019296 --- /dev/null +++ b/distrib-util/requirements-common.txt @@ -0,0 +1,2 @@ +twine +wheel diff --git a/distrib-util/requirements-hints-remover.txt b/distrib-util/requirements-hints-remover.txt new file mode 100644 index 000000000..7c4150335 --- /dev/null +++ b/distrib-util/requirements-hints-remover.txt @@ -0,0 +1 @@ +astor diff --git a/distrib.gradle b/distrib.gradle deleted file mode 100644 index ee88e80e0..000000000 --- a/distrib.gradle +++ /dev/null @@ -1,178 +0,0 @@ -import java.util.function.Consumer - -/****** Prepare distribution ******/ - -task copyDistribFiles(type: Copy, dependsOn: cleanInstallDirDistrib, group: distribGroup) { - from distributionPath - into distribBuildPath - exclude '.idea/**' -} - -task prepareDistributionDir(dependsOn: [cleanInstallDirDistrib, copyDistribFiles], group: distribGroup) { - doLast { - def versionFilePath = distribBuildPath.resolve(versionFileName) - versionFilePath.toFile().write(version as String) - project.copy { - from versionFilePath - into artifactsDir - } - } -} - -createInstallTasks(false, distribBuildPath.resolve(distribKernelDir), distribBuildPath.resolve(runKernelDir)) - -/****** Conda upload ******/ - -task condaPackage(type: Exec, dependsOn: [cleanInstallDirDistrib, preparePackage], group: condaGroup) { - commandLine 'conda-build', 'conda', '--output-folder', condaPackageDir - workingDir distribBuildPath - doLast { - copy { - from distribBuildPath.resolve(condaPackageDir).resolve("noarch").resolve(condaPackageFileName) - into artifactsDir - } - } -} - -class TaskSpec { - String taskName -} - -class UploadTaskSpecs { - T stable - T dev - - Project project - String repoName - String taskGroup - - UploadTaskSpecs(Project project, String repoName, String taskGroup, T stable, T dev) { - this.project = project - this.repoName = repoName - this.taskGroup = taskGroup - this.stable = stable - this.dev = dev - - this.stable.taskName = taskName("Stable") - this.dev.taskName = taskName("Dev") - } - - String taskName(String type) { - return repoName + "Upload" + type - } - - void createTasks(Consumer taskCreationAction) { - if (project.isProtectedBranch) { - taskCreationAction.accept(stable) - } - taskCreationAction.accept(dev) - - project.task([dependsOn: project.cleanInstallDirDistrib, group: taskGroup], taskName("Protected")) { - if (project.isProtectedBranch) { - dependsOn(dev.taskName) - } - } - } -} - -class CondaCredentials { - String username - String password -} - -class CondaTaskSpec extends TaskSpec { - String username - CondaCredentials credentials -} - -ext.condaCredentials = new CondaCredentials( - username: condaUserStable, - password: condaPasswordStable -) - -ext.condaTaskSpecs = new UploadTaskSpecs( - project, "conda", condaGroup, - new CondaTaskSpec( - username: condaUserStable, - credentials: ext.condaCredentials - ), - new CondaTaskSpec( - username: condaUserDev, - credentials: ext.condaCredentials - ) -) - -condaTaskSpecs.createTasks { TaskSpec taskSpec -> - task(taskSpec.taskName, group: condaGroup) { - def artifactPath = artifactsDir.resolve(condaPackageFileName) - - if (!artifactPath.toFile().exists()) { - dependsOn([cleanInstallDirDistrib, condaPackage]) - } - - doLast { - exec { - commandLine 'anaconda', 'login', - '--username', taskSpec.credentials.username, - '--password', taskSpec.credentials.password - standardInput new ByteArrayInputStream("yes".bytes) - } - - exec { - commandLine('anaconda', 'upload', '-u', taskSpec.username, artifactPath.toString()) - } - } - } -} - -/****** PyPi upload ******/ - -task pyPiPackage(type: Exec, dependsOn: preparePackage, group: pyPiGroup) { - commandLine 'python', 'setup.py', 'bdist_wheel', - '--dist-dir', pyPiPackageDir - workingDir distribBuildPath - - doLast { - copy { - from distribBuildPath.resolve(pyPiPackageDir).resolve(pyPiPackageFileName) - into artifactsDir - } - } -} - -class PyPiTaskSpec extends TaskSpec { - String repoURL - String username - String password -} - -def pyPiTaskSpecs = new UploadTaskSpecs( - project, "pyPi", pyPiGroup, - new PyPiTaskSpec( - repoURL: "https://upload.pypi.org/legacy/", - username: stablePyPiUser, - password: stablePyPiPassword - ), - new PyPiTaskSpec( - repoURL: "https://test.pypi.org/legacy/", - username: devPyPiUser, - password: devPyPiPassword - ) -) - -pyPiTaskSpecs.createTasks { taskSpec -> - task([type: Exec, group: pyPiGroup], taskSpec.taskName) { - workingDir(artifactsDir) - def artifactPath = artifactsDir.resolve(pyPiPackageFileName) - - if (!artifactPath.toFile().exists()) { - dependsOn([pyPiPackage]) - } - - commandLine "twine", "upload", - "-u", taskSpec.username, - "-p", taskSpec.password, - "--repository-url", taskSpec.repoURL, - pyPiPackageFileName - } -} diff --git a/distrib/run_kotlin_kernel/run_kernel.py b/distrib/run_kotlin_kernel/run_kernel.py index bc12bb219..41e172b16 100644 --- a/distrib/run_kotlin_kernel/run_kernel.py +++ b/distrib/run_kotlin_kernel/run_kernel.py @@ -9,7 +9,7 @@ def run_kernel(*args) -> None: try: run_kernel_impl(*args) except KeyboardInterrupt: - print("Kernel interrupted") + print('Kernel interrupted') try: sys.exit(130) except SystemExit: @@ -22,29 +22,29 @@ def run_kernel_impl(connection_file: str, jar_args_file: str = None, executables current_dir = os.path.dirname(abspath) if jar_args_file is None: - jar_args_file = os.path.join(current_dir, "config", "jar_args.json") + jar_args_file = os.path.join(current_dir, 'config', 'jar_args.json') if executables_dir is None: executables_dir = current_dir - jars_dir = os.path.join(executables_dir, "jars") + jars_dir = os.path.join(executables_dir, 'jars') - with open(jar_args_file, "r") as fd: + with open(jar_args_file, 'r') as fd: jar_args_json = json.load(fd) - debug: str = jar_args_json["debuggerConfig"] - cp: List[str] = jar_args_json["classPath"] - main_jar: str = jar_args_json["mainJar"] + debug: str = jar_args_json['debuggerConfig'] + cp: List[str] = jar_args_json['classPath'] + main_jar: str = jar_args_json['mainJar'] - debug_list = [] if debug is None or debug == "" else [debug] + debug_list = [] if debug is None or debug == '' else [debug] class_path_arg = os.pathsep.join([os.path.join(jars_dir, jar_name) for jar_name in cp]) main_jar_path = os.path.join(jars_dir, main_jar) - subprocess.call(["java", "-jar"] + debug_list + + subprocess.call(['java', '-jar'] + debug_list + [main_jar_path, - "-classpath=" + class_path_arg, + '-classpath=' + class_path_arg, connection_file, - "-home=" + executables_dir]) + '-home=' + executables_dir]) -if __name__ == "__main__": +if __name__ == '__main__': run_kernel(*(sys.argv[1:])) diff --git a/distrib/setup.py b/distrib/setup.py index 1203cd171..94de86734 100644 --- a/distrib/setup.py +++ b/distrib/setup.py @@ -1,24 +1,35 @@ import glob +import shutil from os import path -from setuptools import setup, find_packages - -abspath = path.abspath(__file__) -current_dir = path.dirname(abspath) -version_file = path.join(current_dir, 'VERSION') +from pathlib import Path -with open(version_file, 'r') as f: - version = f.read().strip() +from setuptools import setup, find_packages DATA_FILES = [ - ('share/jupyter/kernels/kotlin', glob.glob('kernel/*.json')) + ('share/jupyter/kernels/kotlin', glob.glob('kernel/*')) ] - PACKAGE_DATA = { - 'run_kotlin_kernel': ['jars/*.jar', 'config/*.json'] + 'run_kotlin_kernel': ['jars/*.jar', 'config/*.json', 'libraries/*'] } -if __name__ == "__main__": +LIBRARIES_FOLDER_NAME = '.jupyter_kotlin' +LIBRARIES_CACHE_FOLDER_NAME = 'cache' +VERSION_FILE_NAME = 'VERSION' + + +def main(): + home_dir = str(Path.home()) + libraries_cache_path = path.join(home_dir, LIBRARIES_FOLDER_NAME, LIBRARIES_CACHE_FOLDER_NAME) + shutil.rmtree(libraries_cache_path, ignore_errors=True) + + abspath = path.abspath(__file__) + current_dir = path.dirname(abspath) + version_file = path.join(current_dir, VERSION_FILE_NAME) + + with open(version_file, 'r') as f: + version = f.read().strip() + setup(name="kotlin-jupyter-kernel", author="JetBrains", version=version, @@ -29,3 +40,7 @@ package_data=PACKAGE_DATA, data_files=DATA_FILES ) + + +if __name__ == "__main__": + main() diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 000000000..dadd92cff --- /dev/null +++ b/gradle.properties @@ -0,0 +1,7 @@ +# kotlinVersion=1.4.255-SNAPSHOT +kotlinVersion=1.4.0-dev-7568 +kotlinLanguageLevel=1.4 +jvmTarget=1.8 + +shadowJarVersion=5.2.0 +baseVersion=0.7.41 diff --git a/gradle.properties.template b/gradle.properties.local similarity index 93% rename from gradle.properties.template rename to gradle.properties.local index e2883d710..5f9cafc8c 100644 --- a/gradle.properties.template +++ b/gradle.properties.local @@ -3,6 +3,7 @@ installPath=/kernels/kotlin build.branch=dev build.number=1.0.0 build.counter=1 +build.isLocal=true condaUserStable=jetbrains condaPasswordStable= diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b6e49536a..d6ede80f3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-all.zip diff --git a/jupyter-lib/build.gradle b/jupyter-lib/build.gradle deleted file mode 100644 index 770686d11..000000000 --- a/jupyter-lib/build.gradle +++ /dev/null @@ -1 +0,0 @@ -apply plugin: 'kotlin' \ No newline at end of file diff --git a/jupyter-lib/build.gradle.kts b/jupyter-lib/build.gradle.kts new file mode 100644 index 000000000..c14093eef --- /dev/null +++ b/jupyter-lib/build.gradle.kts @@ -0,0 +1,11 @@ +plugins { + kotlin("jvm") +} + +project.version = rootProject.version + +dependencies { + implementation(kotlinDep("kotlin-stdlib")) + testImplementation("junit:junit:4.12") + testImplementation(kotlinDep("kotlin-test")) +} diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinContext.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinContext.kt deleted file mode 100644 index bd4817f7d..000000000 --- a/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinContext.kt +++ /dev/null @@ -1,38 +0,0 @@ -package jupyter.kotlin.completion - -import java.util.* - -/** - * Kotlin REPL has built-in context for getting user-declared functions and variables - * and setting invokeWrapper for additional side effects in evaluation. - * It can be accessed inside REPL by name `kc`, e.g. kc.showVars() - */ -class KotlinContext(val vars: HashMap = HashMap(), - val functions: MutableSet = TreeSet()) { - - fun getVarsList(): List { - return ArrayList(vars.values) - } - - fun getFunctionsList(): List { - return ArrayList(functions) - } -} - - - -/** - * The implicit receiver for lines in Kotlin REPL. - * It is passed to the script as an implicit receiver, identical to: - * with (context) { - * ... - * } - * - * KotlinReceiver can be inherited from and passed to REPL building properties, - * so other variables and functions can be accessed inside REPL. - * By default, it only has KotlinContext. - * Inherited KotlinReceivers should be in separate java file, they can't be inner or nested. - */ -class KotlinReceiver { - var kc: KotlinContext? = null -} diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinFunctionInfo.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinFunctionInfo.kt deleted file mode 100644 index c854c8f6c..000000000 --- a/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinFunctionInfo.kt +++ /dev/null @@ -1,37 +0,0 @@ -package jupyter.kotlin.completion - -import kotlin.reflect.KFunction - -import jupyter.kotlin.completion.KotlinReflectUtil.functionSignature -import jupyter.kotlin.completion.KotlinReflectUtil.shorten - - -class KotlinFunctionInfo(private val function: KFunction<*>) : Comparable { - - val name: String - get() = function.name - - fun toString(shortenTypes: Boolean): String { - return if (shortenTypes) { - shorten(toString()) - } else toString() - } - - override fun toString(): String { - return functionSignature(function) - } - - override fun compareTo(other: KotlinFunctionInfo): Int { - return this.toString().compareTo(other.toString()) - } - - override fun hashCode(): Int { - return this.toString().hashCode() - } - - override fun equals(other: Any?): Boolean { - return if (other is KotlinFunctionInfo) { - this.toString() == other.toString() - } else false - } -} diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinReflectUtil.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinReflectUtil.kt deleted file mode 100644 index 105d1991b..000000000 --- a/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinReflectUtil.kt +++ /dev/null @@ -1,17 +0,0 @@ -package jupyter.kotlin.completion - -import kotlin.reflect.KFunction - -/** - * Util class for pretty-printing Kotlin variables and functions. - */ -object KotlinReflectUtil { - fun functionSignature(function: KFunction<*>): String { - return function.toString().replace("Line_\\d+\\.".toRegex(), "") - } - - fun shorten(name: String): String { - return name.replace("(\\b[_a-zA-Z$][_a-zA-Z0-9$]*\\b\\.)+".toRegex(), "") - // kotlin.collections.List -> List - } -} diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinVariableInfo.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinVariableInfo.kt deleted file mode 100644 index 45eff0d25..000000000 --- a/jupyter-lib/src/main/kotlin/jupyter/kotlin/completion/KotlinVariableInfo.kt +++ /dev/null @@ -1,25 +0,0 @@ -package jupyter.kotlin.completion - -import jupyter.kotlin.completion.KotlinReflectUtil.shorten -import kotlin.reflect.KProperty - -class KotlinVariableInfo(private val value: Any?, private val descriptor: KProperty<*>) { - - val name: String - get() = descriptor.name - - val type: String - get() = descriptor.returnType.toString() - - fun toString(shortenTypes: Boolean): String { - var type: String = type - if (shortenTypes) { - type = shorten(type) - } - return "$name: $type = $value" - } - - override fun toString(): String { - return toString(false) - } -} diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/context.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/context.kt new file mode 100644 index 000000000..144d9dab7 --- /dev/null +++ b/jupyter-lib/src/main/kotlin/jupyter/kotlin/context.kt @@ -0,0 +1,98 @@ +package jupyter.kotlin + +import java.util.* +import kotlin.reflect.KFunction +import kotlin.reflect.KProperty + +/** + * Kotlin REPL has built-in context for getting user-declared functions and variables + * and setting invokeWrapper for additional side effects in evaluation. + * It can be accessed inside REPL by name `kc`, e.g. kc.showVars() + */ +class KotlinContext(val vars: HashMap = HashMap(), + val functions: HashMap = HashMap()) { + + fun getVarsList(): List { + return ArrayList(vars.values) + } + + fun getFunctionsList(): List { + return ArrayList(functions.values) + } +} + + + +/** + * The implicit receiver for lines in Kotlin REPL. + * It is passed to the script as an implicit receiver, identical to: + * with (context) { + * ... + * } + * + * KotlinReceiver can be inherited from and passed to REPL building properties, + * so other variables and functions can be accessed inside REPL. + * By default, it only has KotlinContext. + * Inherited KotlinReceivers should be in separate java file, they can't be inner or nested. + */ +class KotlinReceiver(val kc: KotlinContext) + +fun functionSignature(function: KFunction<*>): String { + return function.toString().replace("Line_\\d+\\.".toRegex(), "") +} + +fun shortenType(name: String): String { + return name.replace("(\\b[_a-zA-Z$][_a-zA-Z0-9$]*\\b\\.)+".toRegex(), "") + // kotlin.collections.List -> List +} + +class KotlinFunctionInfo(val function: KFunction<*>, val line: Any) : Comparable { + + val name: String + get() = function.name + + fun toString(shortenTypes: Boolean): String { + return if (shortenTypes) { + shortenType(toString()) + } else toString() + } + + override fun toString(): String { + return functionSignature(function) + } + + override fun compareTo(other: KotlinFunctionInfo): Int { + return this.toString().compareTo(other.toString()) + } + + override fun hashCode(): Int { + return this.toString().hashCode() + } + + override fun equals(other: Any?): Boolean { + return if (other is KotlinFunctionInfo) { + this.toString() == other.toString() + } else false + } +} + +class KotlinVariableInfo(val value: Any?, val descriptor: KProperty<*>, val line: Any) { + + val name: String + get() = descriptor.name + + val type: String + get() = descriptor.returnType.toString() + + fun toString(shortenTypes: Boolean): String { + var type: String = type + if (shortenTypes) { + type = shortenType(type) + } + return "$name: $type = $value" + } + + override fun toString(): String { + return toString(false) + } +} \ No newline at end of file diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/receivers/ConstReceiver.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/receivers/ConstReceiver.kt new file mode 100644 index 000000000..f380c86a5 --- /dev/null +++ b/jupyter-lib/src/main/kotlin/jupyter/kotlin/receivers/ConstReceiver.kt @@ -0,0 +1,3 @@ +package jupyter.kotlin.receivers + +class ConstReceiver(val value: Int) \ No newline at end of file diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/receivers/TypeProviderReceiver.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/receivers/TypeProviderReceiver.kt new file mode 100644 index 000000000..617468efb --- /dev/null +++ b/jupyter-lib/src/main/kotlin/jupyter/kotlin/receivers/TypeProviderReceiver.kt @@ -0,0 +1,19 @@ +package jupyter.kotlin.receivers + +class TypeProviderReceiver { + + fun generateCode(values: List): List { + val properties = (0 until values.size) + .map { "val value$it : Int get() = list[$it]" } + .joinToString("\n") + + val classDeclaration = """ + class TypedIntList###(val list: List): List by list { + $properties + } + """ + + val converter = "TypedIntList###(\$it)" + return listOf(classDeclaration, converter) + } +} diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/results.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/results.kt index 3b7256067..cdf40904b 100644 --- a/jupyter-lib/src/main/kotlin/jupyter/kotlin/results.kt +++ b/jupyter-lib/src/main/kotlin/jupyter/kotlin/results.kt @@ -14,13 +14,21 @@ annotation class DependsOn(val value: String = "") @Retention(AnnotationRetention.SOURCE) annotation class Repository(val value: String = "") -abstract class ScriptTemplateWithDisplayHelpers { +interface KotlinKernelHost { + fun display(value: Any) + + fun scheduleExecution(code: String) +} + +abstract class ScriptTemplateWithDisplayHelpers(val __host: KotlinKernelHost?) { fun MIME(vararg mimeToData: Pair): MimeTypedResult = MimeTypedResult(mapOf(*mimeToData)) - fun HTML(text: String) = MIME("text/html" to text) + fun HTML(text: String, isolated: Boolean = false) = MIME("text/html" to text).also { it.isolatedHtml = isolated } - fun DISPLAY(value: Any) = DisplayResult(value) + fun DISPLAY(value: Any) = __host!!.display(value) + + fun EXECUTE(code: String) = __host!!.scheduleExecution(code) val Out: List = ReplOutputs } @@ -28,8 +36,6 @@ abstract class ScriptTemplateWithDisplayHelpers { fun mimeResult(vararg mimeToData: Pair): MimeTypedResult = MimeTypedResult(mapOf(*mimeToData)) fun textResult(text: String): MimeTypedResult = MimeTypedResult(mapOf("text/plain" to text)) -class MimeTypedResult(mimeData: Map): Map by mimeData - -class DisplayResult(val value: Any) +class MimeTypedResult(mimeData: Map, var isolatedHtml: Boolean = false) : Map by mimeData val ReplOutputs = mutableListOf() \ No newline at end of file diff --git a/jupyter-lib/src/main/kotlin/jupyter/kotlin/types.kt b/jupyter-lib/src/main/kotlin/jupyter/kotlin/types.kt new file mode 100644 index 000000000..265cb173c --- /dev/null +++ b/jupyter-lib/src/main/kotlin/jupyter/kotlin/types.kt @@ -0,0 +1,12 @@ +package jupyter.kotlin + +sealed class Modificator { + data class AddColumn(val name: String, val getValue: T.() -> R) : Modificator() + data class RemoveColumn(val name: String) : Modificator() +} + +data class ModifiedList(val source: List, val modificator: Modificator) : List by source + +fun List.addColumn(name: String, value: T.() -> R) = ModifiedList(this, Modificator.AddColumn(name, value)) + +fun List.removeColumn(name: String) = ModifiedList(this, Modificator.RemoveColumn(name)) \ No newline at end of file diff --git a/libraries/khttp.json b/libraries/khttp.json new file mode 100644 index 000000000..c887a0c8b --- /dev/null +++ b/libraries/khttp.json @@ -0,0 +1,9 @@ +{ + "properties": { + "v": "1.0.0" + }, + "link": "https://github.com/jkcclemens/khttp", + "dependencies": [ + "khttp:khttp:$v" + ] +} diff --git a/libraries/krangl-typed.json b/libraries/krangl-typed.json new file mode 100644 index 000000000..794be632c --- /dev/null +++ b/libraries/krangl-typed.json @@ -0,0 +1,31 @@ +{ + "init": [ + "fun krangl.DataFrame.toHTML(limit: Int = 20, truncate: Int = 50) : String\n{val sb = StringBuilder()\nsb.append(\"\")\nsb.append(\"\")\ncols.forEach { sb.append(\"\") }\nsb.append(\"\")\nrows.take(limit).forEach {\n sb.append(\"\")\n it.values.map{it.toString()}.forEach { \n val truncated = if (truncate > 0 && it.length > truncate) {\n if (truncate < 4) it.substring(0, truncate)\n else it.substring(0, truncate - 3) + \"...\"\n } else {\n it\n }\n sb.append(\"\"\"\"\"\") \n }\n sb.append(\"\")\n}\nsb.append(\"
${it.name}
$truncated
\")\nif(limit < rows.count())\n sb.append(\"

... only showing top $limit rows

\")\nsb.append(\"\")\nreturn sb.toString()}", + "fun krangl.typed.TypedDataFrame<*>.extractScheme(name: String? = null) = EXECUTE(getScheme(name))" + ], + "imports": [ + "krangl.*", + "krangl.typed.*" + ], + "renderers": { + "krangl.SimpleDataFrame": "HTML($it.toHTML())", + "krangl.typed.TypedDataFrameImpl": "($it as TypedDataFrame<*>).df", + "krangl.typed.TypedDataFrameRowImpl": "($it as TypedDataFrameRow<*>).toDataFrame()" + }, + "repositories": [ + "https://jitpack.io" + ], + "link": "https://github.com/holgerbrandl/krangl", + "dependencies": [ + "com.github.nikitinas:krangl:724fa3613b42a0e5c5d68228ca09a57078e818ba" + ], + "typeConverters": { + "krangl.DataFrame": "CodeGenerator.generate($it, $property)", + "krangl.typed.TypedDataFrame<*>": "CodeGenerator.generate($it, $property)", + "krangl.typed.DataFrameToListNamedStub": "CodeGenerator.generate($it)", + "krangl.typed.DataFrameToListTypedStub": "CodeGenerator.generate($it)" + }, + "annotationHandlers": { + "krangl.typed.DataFrameType": "CodeGenerator.generate($kclass)" + } +} \ No newline at end of file diff --git a/libraries/krangl.json b/libraries/krangl.json index 0d0d77b08..a4c5ba418 100644 --- a/libraries/krangl.json +++ b/libraries/krangl.json @@ -12,10 +12,7 @@ "init": [ "fun krangl.DataFrame.toHTML(limit: Int = 20, truncate: Int = 50) : String\n{val sb = StringBuilder()\nsb.append(\"\")\nsb.append(\"\")\ncols.forEach { sb.append(\"\") }\nsb.append(\"\")\nrows.take(limit).forEach {\n sb.append(\"\")\n it.values.map{it.toString()}.forEach { \n val truncated = if (truncate > 0 && it.length > truncate) {\n if (truncate < 4) it.substring(0, truncate)\n else it.substring(0, truncate - 3) + \"...\"\n } else {\n it\n }\n sb.append(\"\"\"\"\"\") \n }\n sb.append(\"\")\n}\nsb.append(\"
${it.name}
$truncated
\")\nif(limit < rows.count())\n sb.append(\"

... only showing top $limit rows

\")\nsb.append(\"\")\nreturn sb.toString()}" ], - "renderers": [ - { - "class": "krangl.SimpleDataFrame", - "result": "HTML($it.toHTML())" - } - ] + "renderers": { + "krangl.SimpleDataFrame": "HTML($it.toHTML())" + } } diff --git a/libraries/kravis.json b/libraries/kravis.json index c13301f4b..df0fc9a61 100644 --- a/libraries/kravis.json +++ b/libraries/kravis.json @@ -9,10 +9,7 @@ "imports": [ "kravis.*" ], - "renderers": [ - { - "class": "kravis.GGPlot", - "result": "$it.show()" - } - ] + "renderers": { + "kravis.GGPlot": "$it.show()" + } } diff --git a/libraries/lets-plot.json b/libraries/lets-plot.json index a5ae93ee2..f37f06b67 100644 --- a/libraries/lets-plot.json +++ b/libraries/lets-plot.json @@ -22,10 +22,7 @@ "fun jetbrains.letsPlot.intern.Plot.getHtml() = jetbrains.letsPlot.intern.frontendContext.FrontendContextUtil.getHtml(this)", "DISPLAY(HTML(jetbrains.datalore.jupyter.configureScript()))" ], - "renderers": [ - { - "class": "jetbrains.letsPlot.intern.Plot", - "result": "HTML(($it as jetbrains.letsPlot.intern.Plot).getHtml())" - } - ] + "renderers": { + "jetbrains.letsPlot.intern.Plot": "HTML(($it as jetbrains.letsPlot.intern.Plot).getHtml())" + } } diff --git a/libraries/numpy.json b/libraries/numpy.json index b3754482b..6280ba675 100644 --- a/libraries/numpy.json +++ b/libraries/numpy.json @@ -1,6 +1,6 @@ { "properties": { - "v": "0.1.0" + "v": "0.1.1" }, "link": "https://github.com/Kotlin/kotlin-numpy", "repositories": [ diff --git a/libraries/spark.json b/libraries/spark.json index 488a42f27..87fe4ec53 100644 --- a/libraries/spark.json +++ b/libraries/spark.json @@ -37,10 +37,7 @@ "scala.Console.setOut(System.out)", "scala.Console.setErr(System.err)" ], - "renderers": [ - { - "class": "org.apache.spark.sql.Dataset", - "result": "HTML($it.toHTML())" - } - ] + "renderers": { + "org.apache.spark.sql.Dataset": "HTML($it.toHTML())" + } } diff --git a/resources/notebook-extension/kernel.css b/resources/notebook-extension/kernel.css new file mode 100644 index 000000000..7988e1cb7 --- /dev/null +++ b/resources/notebook-extension/kernel.css @@ -0,0 +1,486 @@ +:root { + /* Elevation + * + * We style box-shadows using Material Design's idea of elevation. These particular numbers are taken from here: + * + * https://github.com/material-components/material-components-web + * https://material-components-web.appspot.com/elevation.html + */ + + --md-grey-900: #212121; + --md-grey-400: #bdbdbd; + --md-grey-200: #eeeeee; + --md-blue-500: #2196f3; + + + /* The dark theme shadows need a bit of work, but this will probably also require work on the core layout + * colors used in the theme as well. */ + --jp-shadow-base-lightness: 32; + --jp-shadow-umbra-color: rgba( + var(--jp-shadow-base-lightness), + var(--jp-shadow-base-lightness), + var(--jp-shadow-base-lightness), + 0.2 + ); + --jp-shadow-penumbra-color: rgba( + var(--jp-shadow-base-lightness), + var(--jp-shadow-base-lightness), + var(--jp-shadow-base-lightness), + 0.14 + ); + --jp-shadow-ambient-color: rgba( + var(--jp-shadow-base-lightness), + var(--jp-shadow-base-lightness), + var(--jp-shadow-base-lightness), + 0.12 + ); + --jp-elevation-z0: none; + --jp-elevation-z1: 0px 2px 1px -1px var(--jp-shadow-umbra-color), + 0px 1px 1px 0px var(--jp-shadow-penumbra-color), + 0px 1px 3px 0px var(--jp-shadow-ambient-color); + --jp-elevation-z2: 0px 3px 1px -2px var(--jp-shadow-umbra-color), + 0px 2px 2px 0px var(--jp-shadow-penumbra-color), + 0px 1px 5px 0px var(--jp-shadow-ambient-color); + --jp-elevation-z4: 0px 2px 4px -1px var(--jp-shadow-umbra-color), + 0px 4px 5px 0px var(--jp-shadow-penumbra-color), + 0px 1px 10px 0px var(--jp-shadow-ambient-color); + --jp-elevation-z6: 0px 3px 5px -1px var(--jp-shadow-umbra-color), + 0px 6px 10px 0px var(--jp-shadow-penumbra-color), + 0px 1px 18px 0px var(--jp-shadow-ambient-color); + --jp-elevation-z8: 0px 5px 5px -3px var(--jp-shadow-umbra-color), + 0px 8px 10px 1px var(--jp-shadow-penumbra-color), + 0px 3px 14px 2px var(--jp-shadow-ambient-color); + --jp-elevation-z12: 0px 7px 8px -4px var(--jp-shadow-umbra-color), + 0px 12px 17px 2px var(--jp-shadow-penumbra-color), + 0px 5px 22px 4px var(--jp-shadow-ambient-color); + --jp-elevation-z16: 0px 8px 10px -5px var(--jp-shadow-umbra-color), + 0px 16px 24px 2px var(--jp-shadow-penumbra-color), + 0px 6px 30px 5px var(--jp-shadow-ambient-color); + --jp-elevation-z20: 0px 10px 13px -6px var(--jp-shadow-umbra-color), + 0px 20px 31px 3px var(--jp-shadow-penumbra-color), + 0px 8px 38px 7px var(--jp-shadow-ambient-color); + --jp-elevation-z24: 0px 11px 15px -7px var(--jp-shadow-umbra-color), + 0px 24px 38px 3px var(--jp-shadow-penumbra-color), + 0px 9px 46px 8px var(--jp-shadow-ambient-color); + + /* Borders + * + * The following variables, specify the visual styling of borders in JupyterLab. + */ + + --jp-border-width: 1px; + --jp-border-radius: 2px; + + /* UI Fonts + * + * The UI font CSS variables are used for the typography all of the JupyterLab + * user interface elements that are not directly user generated content. + * + * The font sizing here is done assuming that the body font size of --jp-ui-font-size1 + * is applied to a parent element. When children elements, such as headings, are sized + * in em all things will be computed relative to that body size. + */ + + --jp-ui-font-scale-factor: 1.2; + --jp-ui-font-size0: 0.83333em; + --jp-ui-font-size1: 13px; /* Base font size */ + --jp-ui-font-size2: 1.2em; + --jp-ui-font-size3: 1.44em; + + --jp-ui-font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, + Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; + + /* + * Use these font colors against the corresponding main layout colors. + * In a light theme, these go from dark to light. + */ + + /* Defaults use Material Design specification */ + --jp-ui-font-color0: rgba(0, 0, 0, 1); + --jp-ui-font-color1: rgba(0, 0, 0, 0.87); + --jp-ui-font-color2: rgba(0, 0, 0, 0.54); + --jp-ui-font-color3: rgba(0, 0, 0, 0.38); + + /* + * Use these against the brand/accent/warn/error colors. + * These will typically go from light to darker, in both a dark and light theme. + */ + + --jp-ui-inverse-font-color0: rgba(0, 0, 0, 1); + --jp-ui-inverse-font-color1: rgba(0, 0, 0, 0.8); + --jp-ui-inverse-font-color2: rgba(0, 0, 0, 0.5); + --jp-ui-inverse-font-color3: rgba(0, 0, 0, 0.3); + + /* Content Fonts + * + * Content font variables are used for typography of user generated content. + * + * The font sizing here is done assuming that the body font size of --jp-content-font-size1 + * is applied to a parent element. When children elements, such as headings, are sized + * in em all things will be computed relative to that body size. + */ + + --jp-content-line-height: 1.6; + --jp-content-font-scale-factor: 1.2; + --jp-content-font-size0: 0.83333em; + --jp-content-font-size1: 14px; /* Base font size */ + --jp-content-font-size2: 1.2em; + --jp-content-font-size3: 1.44em; + --jp-content-font-size4: 1.728em; + --jp-content-font-size5: 2.0736em; + + /* This gives a magnification of about 125% in presentation mode over normal. */ + --jp-content-presentation-font-size1: 17px; + + --jp-content-heading-line-height: 1; + --jp-content-heading-margin-top: 1.2em; + --jp-content-heading-margin-bottom: 0.8em; + --jp-content-heading-font-weight: 500; + + /* Defaults use Material Design specification */ + --jp-content-font-color0: rgba(0, 0, 0, 1); + --jp-content-font-color1: rgba(0, 0, 0, 0.87); + --jp-content-font-color2: rgba(0, 0, 0, 0.54); + --jp-content-font-color3: rgba(0, 0, 0, 0.38); + + + --jp-content-font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', + Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', + 'Segoe UI Symbol'; + + /* + * Code Fonts + * + * Code font variables are used for typography of code and other monospaces content. + */ + + --jp-code-font-size: 13px; + --jp-code-line-height: 1.3077; /* 17px for 13px base */ + --jp-code-padding: 5px; /* 5px for 13px base, codemirror highlighting needs integer px value */ + --jp-code-font-family-default: Menlo, Consolas, 'DejaVu Sans Mono', monospace; + --jp-code-font-family: var(--jp-code-font-family-default); + + /* This gives a magnification of about 125% in presentation mode over normal. */ + --jp-code-presentation-font-size: 16px; + + /* may need to tweak cursor width if you change font size */ + --jp-code-cursor-width0: 1.4px; + --jp-code-cursor-width1: 2px; + --jp-code-cursor-width2: 4px; + + /* Layout + * + * The following are the main layout colors use in JupyterLab. In a light + * theme these would go from light to dark. + */ + + --jp-layout-color0: #111111; + + /* Inverse Layout + * + * The following are the inverse layout colors use in JupyterLab. In a light + * theme these would go from dark to light. + */ + + --jp-inverse-layout-color0: white; + --jp-inverse-layout-color1: white; + + /* Brand/accent */ + + + /* State colors (warn, error, success, info) */ + + + /* Cell specific styles */ + + --jp-cell-padding: 5px; + + --jp-cell-collapser-width: 8px; + --jp-cell-collapser-min-height: 20px; + --jp-cell-collapser-not-active-hover-opacity: 0.6; + + --jp-cell-editor-active-background: var(--jp-layout-color0); + + --jp-cell-prompt-width: 64px; + --jp-cell-prompt-font-family: 'Source Code Pro', monospace; + --jp-cell-prompt-letter-spacing: 0px; + --jp-cell-prompt-opacity: 1; + --jp-cell-prompt-not-active-opacity: 1; + + /* A custom blend of MD grey and blue 600 + * See https://meyerweb.com/eric/tools/color-blend/#546E7A:1E88E5:5:hex */ + --jp-cell-inprompt-font-color: #307fc1; + /* A custom blend of MD grey and orange 600 + * https://meyerweb.com/eric/tools/color-blend/#546E7A:F4511E:5:hex */ + --jp-cell-outprompt-font-color: #bf5b3d; + + /* Notebook specific styles */ + + --jp-notebook-padding: 10px; + --jp-notebook-multiselected-color: rgba(33, 150, 243, 0.24); + + /* The scroll padding is calculated to fill enough space at the bottom of the + notebook to show one single-line cell (with appropriate padding) at the top + when the notebook is scrolled all the way to the bottom. We also subtract one + pixel so that no scrollbar appears if we have just one single-line cell in the + notebook. This padding is to enable a 'scroll past end' feature in a notebook. + */ + --jp-notebook-scroll-padding: calc( + 100% - var(--jp-code-font-size) * var(--jp-code-line-height) - + var(--jp-code-padding) - var(--jp-cell-padding) - 1px + ); + + /* Rendermime styles */ + + --jp-rendermime-error-background: rgba(244, 67, 54, 0.28); + --jp-rendermime-table-row-hover-background: rgba(3, 169, 244, 0.2); + + /* Dialog specific styles */ + + --jp-dialog-background: rgba(0, 0, 0, 0.6); + + /* Console specific styles */ + + --jp-console-padding: 10px; + + /* Toolbar specific styles */ + + --jp-toolbar-micro-height: 8px; + --jp-toolbar-box-shadow: 0px 0px 2px 0px rgba(0, 0, 0, 0.8); + --jp-toolbar-header-margin: 4px 4px 0px 4px; + --jp-toolbar-active-background: var(--jp-layout-color0); + + /* Input field styles */ + + --jp-input-active-background: var(--jp-layout-color0); + --jp-input-active-box-shadow-color: rgba(19, 124, 189, 0.3); + + /* General editor styles */ + + --jp-editor-selected-focused-background: rgba(33, 150, 243, 0.24); + --jp-editor-cursor-color: var(--jp-ui-font-color0); + + /* Code mirror specific styles */ + + --jp-mirror-editor-operator-color: #aa22ff; + --jp-mirror-editor-comment-color: #408080; + --jp-mirror-editor-string-color: #ba2121; + --jp-mirror-editor-meta-color: #aa22ff; + --jp-mirror-editor-qualifier-color: #555; + --jp-mirror-editor-bracket-color: #997; + --jp-mirror-editor-error-color: #f00; + --jp-mirror-editor-hr-color: #999; + + /* Sidebar-related styles */ + + --jp-sidebar-min-width: 180px; + + /* Search-related styles */ + + --jp-search-toggle-off-opacity: 0.6; + --jp-search-toggle-hover-opacity: 0.8; + --jp-search-toggle-on-opacity: 1; + --jp-search-selected-match-background-color: rgb(255, 225, 0); + --jp-search-selected-match-color: black; + --jp-search-unselected-match-background-color: var( + --jp-inverse-layout-color0 + ); + --jp-search-unselected-match-color: var(--jp-ui-inverse-font-color0); + + /* scrollbar related styles. Supports every browser except Edge. */ + + /* colors based on JetBrain's Darcula theme */ + + --jp-scrollbar-background-color: #3f4244; + --jp-scrollbar-thumb-color: 88, 96, 97; /* need to specify thumb color as an RGB triplet */ + + --jp-scrollbar-endpad: 3px; /* the minimum gap between the thumb and the ends of a scrollbar */ + + /* hacks for setting the thumb shape. These do nothing in Firefox */ + + --jp-scrollbar-thumb-margin: 3.5px; /* the space in between the sides of the thumb and the track */ + --jp-scrollbar-thumb-radius: 9px; /* set to a large-ish value for rounded endcaps on the thumb */ + + + --jp-layout-color1: white; + --jp-border-color1: var(--md-grey-400); + --jp-layout-color2: var(--md-grey-200); + --jp-brand-color1: var(--md-blue-500); + + --jp-private-completer-item-height: 22px; + /* Shift the baseline of the type character to align with the match text */ + --jp-private-completer-type-offset: 2px; +} + +#site { + position: relative !important; +} + +.completions { + /*position: relative !important;*/ +} + +.jp-Completer { + box-shadow: var(--jp-elevation-z6); + background: var(--jp-layout-color1); + color: var(--jp-content-font-color1); + border: var(--jp-border-width) solid var(--jp-border-color1); + list-style-type: none; + overflow-y: scroll; + overflow-x: auto; + padding: 0; + /* Position the completer relative to the text editor, align the '.' */ + margin: 0 0 0 0; + max-height: calc( + (10 * var(--jp-private-completer-item-height)) + + (2 * var(--jp-border-width)) + ); + min-height: calc( + var(--jp-private-completer-item-height) + (2 * var(--jp-border-width)) + ); + z-index: 10001; +} + +.jp-Completer-item { + display: table-row; + box-sizing: border-box; + margin: 0; + padding: 0; + height: var(--jp-private-completer-item-height); + min-width: 150px; +} + +.jp-Completer-item .jp-Completer-match { + display: table-cell; + box-sizing: border-box; + margin: 0; + padding: 0 8px 0 6px; + height: var(--jp-private-completer-item-height); + font-family: var(--jp-code-font-family); + font-size: var(--jp-code-font-size); + line-height: var(--jp-private-completer-item-height); +} + +.jp-Completer-item .jp-Completer-type { + display: table-cell; + box-sizing: border-box; + height: var(--jp-private-completer-item-height); + text-align: center; + background: transparent; + color: white; + width: var(--jp-private-completer-item-height); + font-family: var(--jp-ui-font-family); + font-size: var(--jp-ui-font-size1); + line-height: calc( + var(--jp-private-completer-item-height) - + var(--jp-private-completer-type-offset) + ); + padding-bottom: var(--jp-private-completer-type-offset); +} + +.jp-Completer-item .jp-Completer-typeExtended { + display: table-cell; + box-sizing: border-box; + height: var(--jp-private-completer-item-height); + text-align: right; + background: transparent; + color: var(--jp-ui-font-color2); + font-family: var(--jp-code-font-family); + font-size: var(--jp-code-font-size); + line-height: var(--jp-private-completer-item-height); + padding-right: 8px; +} + +.jp-Completer-item:hover { + background: var(--jp-layout-color2); + opacity: 0.8; +} + +.jp-Completer-item.jp-mod-active { + background: var(--jp-brand-color1); + color: white; +} + +.jp-Completer-item .jp-Completer-match mark { + font-weight: bold; + background: inherit; + color: inherit; +} + +.jp-Completer-type[data-color-index='0'] { + background: transparent; +} + +.jp-Completer-type[data-color-index='1'] { + background: #1f77b4; +} + +.jp-Completer-type[data-color-index='2'] { + background: #ff7f0e; +} + +.jp-Completer-type[data-color-index='3'] { + background: #2ca02c; +} + +.jp-Completer-type[data-color-index='4'] { + background: #d62728; +} + +.jp-Completer-type[data-color-index='5'] { + background: #9467bd; +} + +.jp-Completer-type[data-color-index='6'] { + background: #8c564b; +} + +.jp-Completer-type[data-color-index='7'] { + background: #e377c2; +} + +.jp-Completer-type[data-color-index='8'] { + background: #7f7f7f; +} + +.jp-Completer-type[data-color-index='9'] { + background: #bcbd22; +} + +.jp-Completer-type[data-color-index='10'] { + background: #17becf; +} + +.cm__red_wavy_line { + background: url("/kernelspecs/kotlin/wavyline-red.gif") repeat-x 100% 100%; + padding-bottom: 2px; +} + +.cm__orange_wavy_line { + background: url("/kernelspecs/kotlin/wavyline-orange.gif") repeat-x 100% 100%; + padding-bottom: 2px; +} + +.kotlin-error-tooltip { + border: 1px solid silver; + border-radius: 3px; + color: #444; + padding: 2px 5px; + font-size: 90%; + font-family: monospace; + background-color: white; + white-space: pre-wrap; + + max-width: 40em; + position: absolute; + z-index: 10; + -webkit-box-shadow: 2px 3px 5px rgba(0,0,0,.2); + -moz-box-shadow: 2px 3px 5px rgba(0,0,0,.2); + box-shadow: 2px 3px 5px rgba(0,0,0,.2); + + transition: opacity 1s; + -moz-transition: opacity 1s; + -webkit-transition: opacity 1s; + -o-transition: opacity 1s; + -ms-transition: opacity 1s; +} diff --git a/resources/notebook-extension/kernel.js b/resources/notebook-extension/kernel.js new file mode 100644 index 000000000..cf1d1bca3 --- /dev/null +++ b/resources/notebook-extension/kernel.js @@ -0,0 +1,968 @@ +define(function(){ + function onload() { + if (!Element.prototype.scrollIntoViewIfNeeded) { + Element.prototype.scrollIntoViewIfNeeded = function (centerIfNeeded) { + centerIfNeeded = arguments.length === 0 ? true : !!centerIfNeeded; + + var parent = this.parentNode, + parentComputedStyle = window.getComputedStyle(parent, null), + parentBorderTopWidth = parseInt(parentComputedStyle.getPropertyValue('border-top-width')), + parentBorderLeftWidth = parseInt(parentComputedStyle.getPropertyValue('border-left-width')), + overTop = this.offsetTop - parent.offsetTop < parent.scrollTop, + overBottom = (this.offsetTop - parent.offsetTop + this.clientHeight - parentBorderTopWidth) > (parent.scrollTop + parent.clientHeight), + overLeft = this.offsetLeft - parent.offsetLeft < parent.scrollLeft, + overRight = (this.offsetLeft - parent.offsetLeft + this.clientWidth - parentBorderLeftWidth) > (parent.scrollLeft + parent.clientWidth), + alignWithTop = overTop && !overBottom; + + if ((overTop || overBottom) && centerIfNeeded) { + parent.scrollTop = this.offsetTop - parent.offsetTop - parent.clientHeight / 2 - parentBorderTopWidth + this.clientHeight / 2; + } + + if ((overLeft || overRight) && centerIfNeeded) { + parent.scrollLeft = this.offsetLeft - parent.offsetLeft - parent.clientWidth / 2 - parentBorderLeftWidth + this.clientWidth / 2; + } + + if ((overTop || overBottom || overLeft || overRight) && !centerIfNeeded) { + this.scrollIntoView(alignWithTop); + } + }; + } + + var utils = require('base/js/utils'); + var keyboard = require('base/js/keyboard'); + var $ = require('jquery'); + var CodeMirror = require('codemirror/lib/codemirror'); + var Completer = requirejs("notebook/js/completer").Completer; + var Cell = requirejs("notebook/js/cell").Cell; + var CodeCell = requirejs("notebook/js/codecell").CodeCell; + var Kernel = requirejs("services/kernels/kernel").Kernel; + + var error_class = "cm__red_wavy_line"; + var warning_class = "cm__orange_wavy_line"; + var additionalClasses = [error_class, warning_class]; + var diag_class = { + "ERROR": error_class, + "WARNING": warning_class + }; + + function isOnScreen(elem) { + var bounding = elem.getBoundingClientRect(); + return ( + bounding.top >= 0 && + bounding.left >= 0 && + bounding.bottom <= (window.innerHeight || document.documentElement.clientHeight) && + bounding.right <= (window.innerWidth || document.documentElement.clientWidth) + ); + } + + var opened_completer = null; + var siteEl = document.getElementById("site"); + $(siteEl).scroll((e) => { + if (opened_completer && !isOnScreen(opened_completer.complete[0])) { + opened_completer.close(); + } + }); + + var cssUrl = require.toUrl(Jupyter.kernelselector.kernelspecs.kotlin.resources["kernel.css"]); + $('head').append(''); + + var keycodes = keyboard.keycodes; + + var prepend_n_prc = function(str, n) { + for( var i =0 ; i< n ; i++){ + str = '%'+str ; + } + return str; + }; + + var _existing_completion = function(item, completion_array){ + for( var i=0; i < completion_array.length; i++) { + if (completion_array[i].trim().substr(-item.length) == item) { + return true; + } + } + return false; + }; + + // what is the common start of all completions + function shared_start(B, drop_prct) { + if (B.length == 1) { + return B[0]; + } + var A = []; + var common; + var min_lead_prct = 10; + for (var i = 0; i < B.length; i++) { + var str = B[i].str; + var localmin = 0; + if(drop_prct === true){ + while ( str.substr(0, 1) == '%') { + localmin = localmin+1; + str = str.substring(1); + } + } + min_lead_prct = Math.min(min_lead_prct, localmin); + A.push(str); + } + + if (A.length > 1) { + var tem1, tem2, s; + A = A.slice(0).sort(); + tem1 = A[0]; + s = tem1.length; + tem2 = A.pop(); + while (s && tem2.indexOf(tem1) == -1) { + tem1 = tem1.substring(0, --s); + } + if (tem1 === "" || tem2.indexOf(tem1) !== 0) { + return { + replaceText: prepend_n_prc('', min_lead_prct), + type: "computed", + from: B[0].from, + to: B[0].to + }; + } + return { + replaceText: prepend_n_prc(tem1, min_lead_prct), + type: "computed", + from: B[0].from, + to: B[0].to + }; + } + return null; + } + + Completer.prototype.startCompletion = function (doAutoPrint) { + /** + * call for a 'first' completion, that will set the editor and do some + * special behavior like autopicking if only one completion available. + */ + this.do_auto_print = !!doAutoPrint; + if (this.editor.somethingSelected()|| this.editor.getSelections().length > 1) return; + this.done = false; + // use to get focus back on opera + this.carry_on_completion(true); + }; + + function indexOf(str, filter, from, to, step) { + var cond = (from < to) ? (j => j <= to) : (j => j >= to); + for (var i = from; cond(i); i += step) { + if (filter(str[i])) + return i; + } + return -1; + } + + function getTokenBounds(buf, cursor, editor) { + if (cursor > buf.length) { + throw new Error("Position " + cursor + " does not exist in code snippet <" + buf + ">"); + } + + var filter = c => !/^[A-Z0-9_]$/i.test(c); + + var start = indexOf(buf, filter, cursor - 1, 0, -1) + 1; + var end = cursor + + return { + before: buf.substring(0, start), + token: buf.substring(start, end), + tokenBeforeCursor: buf.substring(start, cursor), + after: buf.substring(end, buf.length), + start: start, + end: end, + posStart: editor.posFromIndex(start), + posEnd: editor.posFromIndex(end) + } + } + + Completer.prototype.carry_on_completion = function (first_invocation) { + /** + * Pass true as parameter if you want the completer to autopick when + * only one completion. This function is automatically reinvoked at + * each keystroke with first_invocation = false + */ + var cur = this.editor.getCursor(); + var line = this.editor.getLine(cur.line); + var pre_cursor = this.editor.getRange({ + line: cur.line, + ch: cur.ch - 1 + }, cur); + + // we need to check that we are still on a word boundary + // because while typing the completer is still reinvoking itself + // so dismiss if we are on a "bad" character + if (!this.reinvoke(pre_cursor) && !first_invocation) { + this.close(); + return; + } + + this.autopick = !!first_invocation; + + // We want a single cursor position. + if (this.editor.somethingSelected()|| this.editor.getSelections().length > 1) { + return; + } + + var cursor_pos = this.editor.indexFromPos(cur); + var text = this.editor.getValue(); + cursor_pos = utils.js_idx_to_char_idx(cursor_pos, text); + + var prevBounds = this.tokenBounds; + var bounds = getTokenBounds(text, cursor_pos, this.editor); + this.tokenBounds = bounds; + if (prevBounds && this.raw_result) { + if (bounds.before === prevBounds.before && + bounds.after === prevBounds.after && + bounds.end > prevBounds.end) { + + var newResult = this.raw_result.filter((v) => { + var displayName = v.str; + if (displayName[0] === '`') + displayName = displayName.substring(1, displayName.length - 1); + return displayName.startsWith(bounds.tokenBeforeCursor) + }).map((completion) => { + completion.from = bounds.posStart; + completion.to = bounds.posEnd; + return completion; + }); + + if (newResult.length > 0) { + this.raw_result = newResult; + this.make_gui(this.prepare_cursor_pos(bounds.start, cursor_pos)); + return; + } + } + } + + // one kernel completion came back, finish_completing will be called with the results + // we fork here and directly call finish completing if kernel is busy + + if (this.skip_kernel_completion) { + this.finish_completing({ content: { + matches: [], + cursor_start: cursor_pos, + cursor_end: cursor_pos, + }}); + } else { + this.cell.kernel.complete(text, cursor_pos, + $.proxy(this.finish_completing, this) + ); + } + }; + + function convertPos(pos) { + return { + line: pos.line - 1, + ch: pos.col - 1 + } + } + + function adjustRange(start, end) { + var s = convertPos(start); + var e = convertPos(end); + if (s.line === e.line && s.ch === e.ch) { + s.ch --; + } + return { + start: s, + end: e + } + } + + function highlightErrors (errors, cell) { + errors = errors || []; + errors.forEach(error => { + var start = error.start; + var end = error.end; + if (!start || !end) + return; + var r = adjustRange(start, end); + error.range = r; + var cl = diag_class[error.severity]; + cell.code_mirror.markText(r.start, r.end, {className: cl}); + }); + + cell.errorsList = errors; + } + + Cell.prototype.highlightErrors = function (errors) { + highlightErrors(errors, this) + }; + + Completer.prototype.make_gui = function(cur_pos, cur) { + var start = cur_pos.start; + cur = cur || this.editor.getCursor(); + + // if empty result return + if (!this.raw_result || !this.raw_result.length) { + this.close(); + return; + } + + // When there is only one completion, use it directly. + if (this.do_auto_print && this.autopick && this.raw_result.length == 1) { + this.insert(this.raw_result[0]); + return; + } + + if (this.raw_result.length == 1) { + // test if first and only completion totally matches + // what is typed, in this case dismiss + var str = this.raw_result[0].str; + var pre_cursor = this.editor.getRange({ + line: cur.line, + ch: cur.ch - str.length + }, cur); + if (pre_cursor == str) { + this.close(); + return; + } + } + + if (!this.visible) { + this.complete = $('
').addClass('completions'); + this.complete.attr('id', 'complete'); + + // Currently webkit doesn't use the size attr correctly. See: + // https://code.google.com/p/chromium/issues/detail?id=4579 + this.sel = $('
    ') + .attr('tabindex', -1) + .attr('multiple', 'true') + .addClass('jp-Completer'); + this.complete.append(this.sel); + this.visible = true; + $(siteEl).append(this.complete); + + //build the container + var that = this; + this._handle_keydown = function (cm, event) { + that.keydown(event); + }; + this.editor.on('keydown', this._handle_keydown); + this._handle_keypress = function (cm, event) { + that.keypress(event); + }; + this.editor.on('keypress', this._handle_keypress); + } + this.sel.attr('size', Math.min(10, this.raw_result.length)); + + // Clear and fill the list. + this.sel.text(''); + this.build_gui_list(this.raw_result); + + // After everything is on the page, compute the position. + // We put it above the code if it is too close to the bottom of the page. + var pos = this.editor.cursorCoords( + this.editor.posFromIndex(start) + ); + var left = pos.left-3; + var top; + var cheight = this.complete.height(); + var wheight = $(window).height(); + if (pos.bottom+cheight+5 > wheight) { + top = pos.top-cheight-4; + } else { + top = pos.bottom+1; + } + + this.complete.css('left', left + siteEl.scrollLeft - siteEl.offsetLeft + 'px'); + this.complete.css('top', top + siteEl.scrollTop - siteEl.offsetTop + 'px'); + + opened_completer = this; + return true; + }; + + Completer.prototype.prepare_cursor_pos = function(start, end) { + if (end === null) { + // adapted message spec replies don't have cursor position info, + // interpret end=null as current position, + // and negative start relative to that + end = this.editor.indexFromPos(cur); + if (start === null) { + start = end; + } else if (start < 0) { + start = end + start; + } + } else { + // handle surrogate pairs + var text = this.editor.getValue(); + end = utils.char_idx_to_js_idx(end, text); + start = utils.char_idx_to_js_idx(start, text); + } + + return { + start: start, + end: end + } + }; + + Completer.prototype.finish_completing = function (msg) { + /** + * let's build a function that wrap all that stuff into what is needed + * for the new completer: + */ + + // alert("WOW"); + + var content = msg.content; + var start = content.cursor_start; + var end = content.cursor_end; + var matches = content.matches; + var metadata = content.metadata || {}; + var extMetadata = metadata._jupyter_extended_metadata || {}; + + console.log(content); + + var cur = this.editor.getCursor(); + + var paragraph = content.paragraph; + if (paragraph) { + if (paragraph.cursor !== this.editor.indexFromPos(cur) + || paragraph.text !== this.editor.getValue()) { + // this.close(); + return; + } + } + + var newPos = this.prepare_cursor_pos(start, end); + start = newPos.start; + end = newPos.end; + + var filtered_results = []; + //remove results from context completion + //that are already in kernel completion + var i; + + // append the introspection result, in order, at at the beginning of + // the table and compute the replacement range from current cursor + // position and matched_text length. + var from = this.editor.posFromIndex(start); + var to = this.editor.posFromIndex(end); + for (i = matches.length - 1; i >= 0; --i) { + var info = extMetadata[i] || {}; + var replaceText = info.text || matches[i]; + var displayText = info.displayText || replaceText; + + filtered_results.unshift({ + str: displayText, + replaceText: replaceText, + tail: info.tail, + icon: info.icon, + type: "introspection", + from: from, + to: to + }); + } + + // one the 2 sources results have been merge, deal with it + this.raw_result = filtered_results; + + this.make_gui(newPos, cur); + }; + + Completer.prototype.pickColor = function(iconText) { + this.colorsDict = this.colorsDict || {}; + var colorInd = this.colorsDict[iconText]; + if (colorInd) { + return colorInd; + } + colorInd = Math.floor(Math.random() * 10) + 1; + this.colorsDict[iconText] = colorInd; + return colorInd; + }; + + Completer.prototype.insert = function (completion) { + this.editor.replaceRange(completion.replaceText, completion.from, completion.to); + }; + + Completer.prototype.build_gui_list = function (completions) { + var MAXIMUM_GUI_LIST_LENGTH = 1000; + var that = this; + for (var i = 0; i < completions.length && i < MAXIMUM_GUI_LIST_LENGTH; ++i) { + var comp = completions[i]; + + var icon = comp.icon || 'u'; + var text = comp.replaceText || ''; + var displayText = comp.str || ''; + var tail = comp.tail || ''; + + var typeColorIndex = this.pickColor(icon); + var iconTag = $('') + .text(icon.charAt(0)) + .addClass('jp-Completer-type') + .attr('data-color-index', typeColorIndex); + + var matchTag = $('').text(displayText).addClass('jp-Completer-match'); + var typeTag = $('').text(tail).addClass('jp-Completer-typeExtended'); + + var opt = $('
  • ').addClass('jp-Completer-item'); + opt.click((function (k, event) { + this.selIndex = k; + this.pick(); + this.editor.focus(); + }).bind(that, i)); + + opt.append(iconTag, matchTag, typeTag); + + this.sel.append(opt); + } + this.sel.children().first().addClass('jp-mod-active'); + this.selIndex = 0; + // this.sel.scrollTop(0); + }; + + Completer.prototype.close = function () { + this.done = true; + opened_completer = null; + $('#complete').remove(); + this.editor.off('keydown', this._handle_keydown); + this.editor.off('keypress', this._handle_keypress); + this.visible = false; + }; + + Completer.prototype.pick = function () { + var ind = this.selIndex; + var completion = this.raw_result[ind]; + this.insert(completion); + this.close(); + }; + + Completer.prototype.selectNew = function(newIndex) { + $(this.sel.children()[this.selIndex]).removeClass('jp-mod-active'); + this.selIndex = newIndex; + + var active = this.sel.children()[this.selIndex]; + $(active).addClass('jp-mod-active'); + active.scrollIntoViewIfNeeded(false); + }; + + Completer.prototype.keydown = function (event) { + var code = event.keyCode; + + // Enter + var optionsLen; + var index; + var prevIndex; + if (code == keycodes.enter && !event.shiftKey) { + event.codemirrorIgnore = true; + event._ipkmIgnore = true; + event.preventDefault(); + this.pick(); + // Escape or backspace + } else if (code == keycodes.esc) { + event.codemirrorIgnore = true; + event._ipkmIgnore = true; + event.preventDefault(); + this.close(); + } else if (code == keycodes.tab) { + //all the fastforwarding operation, + //Check that shared start is not null which can append with prefixed completion + // like %pylab , pylab have no shred start, and ff will result in py + // to erase py + + var sh = shared_start(this.raw_result, true); + if (sh.str !== '') { + this.insert(sh); + } + this.close(); + this.carry_on_completion(); + + // event.codemirrorIgnore = true; + // event._ipkmIgnore = true; + // event.preventDefault(); + // this.pick(); + // this.close(); + + } else if (code == keycodes.up || code == keycodes.down) { + // need to do that to be able to move the arrow + // when on the first or last line ofo a code cell + event.codemirrorIgnore = true; + event._ipkmIgnore = true; + event.preventDefault(); + + optionsLen = this.raw_result.length; + index = this.selIndex; + if (code == keycodes.up) { + index--; + } + if (code == keycodes.down) { + index++; + } + index = Math.min(Math.max(index, 0), optionsLen-1); + this.selectNew(index); + } else if (code == keycodes.pageup || code == keycodes.pagedown) { + event.codemirrorIgnore = true; + event._ipkmIgnore = true; + + optionsLen = this.raw_result.length; + index = this.selIndex; + if (code == keycodes.pageup) { + index -= 10; // As 10 is the hard coded size of the drop down menu + } else { + index += 10; + } + index = Math.min(Math.max(index, 0), optionsLen-1); + this.selectNew(index); + } else if (code == keycodes.left || code == keycodes.right) { + this.close(); + } + }; + + function _isCompletionKey(key) { + return /^[A-Z0-9.:"]$/i.test(key); + } + + Completer.prototype.keypress = function (event) { + /** + * FIXME: This is a band-aid. + * on keypress, trigger insertion of a single character. + * This simulates the old behavior of completion as you type, + * before events were disconnected and CodeMirror stopped + * receiving events while the completer is focused. + */ + + var that = this; + var code = event.keyCode; + + // don't handle keypress if it's not a character (arrows on FF) + // or ENTER/TAB + if (event.charCode === 0 || + code == keycodes.tab || + code == keycodes.enter || + code == keycodes.backspace + ) return; + + if (_isCompletionKey(event.key)) + return; + + // this.close(); + this.editor.focus(); + + setTimeout(function () { + that.carry_on_completion(); + }, 10); + }; + + var EMPTY_ERRORS_RESULT = [[], []]; + + CodeCell.prototype.findErrorsAtPos = function(pos) { + if (pos.outside || Math.abs(pos.xRel) > 10) + return EMPTY_ERRORS_RESULT; + + var ind = this.code_mirror.indexFromPos(pos); + if (!this.errorsList) + return EMPTY_ERRORS_RESULT; + + var filter = (er) => { + var er_start_ind = this.code_mirror.indexFromPos(er.range.start); + var er_end_ind = this.code_mirror.indexFromPos(er.range.end); + return er_start_ind <= ind && ind <= er_end_ind; + }; + var passed = []; + var other = []; + this.errorsList.forEach((er) => { + if (filter(er)) { + passed.push(er); + } else { + other.push(er); + } + }); + return [passed, other]; + }; + + function clearErrors(cm) { + cm.getAllMarks() + .filter(it => additionalClasses.some((cl) => it.className === cl)) + .forEach(it => it.clear()); + } + + function clearAllErrors(notebook) { + notebook.get_cells().forEach((cell) =>{ + if (cell.code_mirror) { + clearErrors(cell.code_mirror); + } + }); + } + + CodeCell.prototype._handle_change = function(cm, changes) { + clearAllErrors(this.notebook); + this.kernel.listErrors(cm.getValue(), (msg) => { + var content = msg.content; + console.log(content); + + if(content.code !== cm.getValue()) { + return; + } + + var errors = content.errors; + this.highlightErrors(errors); + this.errorsList = errors; + }); + }; + + CodeCell.prototype._handle_move = function(e) { + var rect = e.currentTarget.getBoundingClientRect(); + var x = e.clientX - rect.left; + var y = e.clientY - rect.top; + var cursor_pos = this.code_mirror.coordsChar({left: x, top: y}, "local"); + var res = this.findErrorsAtPos(cursor_pos); + var errors = res[0]; + var otherErrors = res[1]; + errors.forEach((error) => { + tempTooltip(this.code_mirror, error, cursor_pos); + }); + otherErrors.forEach((error) => { + if (error.tip) { + remove(error.tip); + } + }) + }; + + + CodeCell.prototype._isCompletionEvent = function(event, cur, editor) { + if (event.type !== 'keydown' || event.ctrlKey || event.metaKey || event.altKey || !this.tooltip._hidden) + return false; + if (event.keyCode === keycodes.tab) + return true; + if (event.keyCode === keycodes.backspace){ + var pre_cursor = editor.getRange({line:0,ch:0},cur); + return pre_cursor.length > 1 && _isCompletionKey(pre_cursor[pre_cursor.length - 2]); + } + return _isCompletionKey(event.key); + }; + + CodeCell.prototype.addEvent = function(obj, event, listener, bind_listener_name) { + if (this[bind_listener_name]) { + return; + } + var handler = this[bind_listener_name] = listener.bind(this); + CodeMirror.off(obj, event, handler); + CodeMirror.on(obj, event, handler); + }; + + CodeCell.prototype.bindEvents = function() { + this.addEvent(this.code_mirror, 'changes', this._handle_change, "binded_handle_change"); + this.addEvent(this.code_mirror.display.lineSpace, 'mousemove', this._handle_move, 'binded_handle_move'); + }; + + CodeCell.prototype.handle_codemirror_keyevent = function (editor, event) { + var that = this; + this.bindEvents(); + + // whatever key is pressed, first, cancel the tooltip request before + // they are sent, and remove tooltip if any, except for tab again + var tooltip_closed = null; + if (event.type === 'keydown' && event.which !== keycodes.tab ) { + tooltip_closed = this.tooltip.remove_and_cancel_tooltip(); + } + + var cur = editor.getCursor(); + if (event.keyCode === keycodes.enter){ + this.auto_highlight(); + } + + if (event.which === keycodes.down && event.type === 'keypress' && this.tooltip.time_before_tooltip >= 0) { + // triger on keypress (!) otherwise inconsistent event.which depending on plateform + // browser and keyboard layout ! + // Pressing '(' , request tooltip, don't forget to reappend it + // The second argument says to hide the tooltip if the docstring + // is actually empty + this.tooltip.pending(that, true); + } else if ( tooltip_closed && event.which === keycodes.esc && event.type === 'keydown') { + // If tooltip is active, cancel it. The call to + // remove_and_cancel_tooltip above doesn't pass, force=true. + // Because of this it won't actually close the tooltip + // if it is in sticky mode. Thus, we have to check again if it is open + // and close it with force=true. + if (!this.tooltip._hidden) { + this.tooltip.remove_and_cancel_tooltip(true); + } + // If we closed the tooltip, don't let CM or the global handlers + // handle this event. + event.codemirrorIgnore = true; + event._ipkmIgnore = true; + event.preventDefault(); + return true; + } else if (event.keyCode === keycodes.tab && event.type === 'keydown' && event.shiftKey) { + if (editor.somethingSelected() || editor.getSelections().length !== 1){ + var anchor = editor.getCursor("anchor"); + var head = editor.getCursor("head"); + if( anchor.line !== head.line){ + return false; + } + } + var pre_cursor = editor.getRange({line:cur.line,ch:0},cur); + if (pre_cursor.trim() === "") { + // Don't show tooltip if the part of the line before the cursor + // is empty. In this case, let CodeMirror handle indentation. + return false; + } + this.tooltip.request(that); + event.codemirrorIgnore = true; + event.preventDefault(); + return true; + } else if (this._isCompletionEvent(event, cur, editor)) { + // Tab completion. + this.tooltip.remove_and_cancel_tooltip(); + + // completion does not work on multicursor, it might be possible though in some cases + if (editor.somethingSelected() || editor.getSelections().length > 1) { + return false; + } + var pre_cursor = editor.getRange({line:cur.line,ch:0},cur); + if (pre_cursor.trim() === "" && event.keyCode === keycodes.tab) { + // Don't autocomplete if the part of the line before the cursor + // is empty. In this case, let CodeMirror handle indentation. + return false; + } else { + event.preventDefault(); + event.codemirrorIgnore = true; + + var doAutoPrint = event.keyCode === keycodes.tab; + + if (!doAutoPrint && event.key.length === 1) { + editor.replaceRange(event.key, cur, cur); + } else if (event.keyCode === keycodes.backspace) { + var fromInd = this.code_mirror.indexFromPos(cur) - 1; + + if (fromInd >= 0) { + editor.replaceRange("", this.code_mirror.posFromIndex(fromInd), cur); + } + } + + this.completer.startCompletion(doAutoPrint); + return true; + } + } + + // keyboard event wasn't one of those unique to code cells, let's see + // if it's one of the generic ones (i.e. check edit mode shortcuts) + return Cell.prototype.handle_codemirror_keyevent.apply(this, [editor, event]); + }; + + CodeCell.prototype._handle_execute_reply = function (msg) { + clearAllErrors(this.notebook) + this.bindEvents(); + + this.set_input_prompt(msg.content.execution_count); + this.element.removeClass("running"); + this.events.trigger('set_dirty.Notebook', {value: true}); + + if (msg.content.status === 'error') { + var addInfo = msg.content.additionalInfo || {}; + var from = {line: addInfo.lineStart, col: addInfo.colStart}; + var to; + if (addInfo.lineEnd !== -1 && addInfo.colEnd !== -1) { + to = {line: addInfo.lineEnd, col: addInfo.colEnd}; + } else { + to = {line: from.line, col: from.col + 3}; + } + + if (from.line !== undefined && from.col !== undefined) { + var message = addInfo.message; + message = message.replace(/^\(\d+:\d+ - \d+\) /, ""); + message = message.replace(/^\(\d+:\d+\) - \(\d+:\d+\) /, ""); + + this.errorsList = [ + { + start: from, + end: to, + message: message, + severity: "ERROR" + }]; + this.highlightErrors(this.errorsList); + } + } + }; + + function tempTooltip(cm, error, pos) { + if (cm.state.errorTip) remove(cm.state.errorTip); + var where = cm.charCoords(pos); + var tip = makeTooltip(where.right + 1, where.bottom, error.message, error.tip); + error.tip = cm.state.errorTip = tip; + fadeIn(tip); + + function maybeClear() { + old = true; + if (!mouseOnTip) clear(); + } + function clear() { + cm.state.ternTooltip = null; + if (tip.parentNode) fadeOut(tip); + clearActivity(); + } + var mouseOnTip = false, old = false; + CodeMirror.on(tip, "mousemove", function() { mouseOnTip = true; }); + CodeMirror.on(tip, "mouseout", function(e) { + var related = e.relatedTarget || e.toElement; + if (!related || !CodeMirror.contains(tip, related)) { + if (old) clear(); + else mouseOnTip = false; + } + }); + setTimeout(maybeClear, 100000); + var clearActivity = onEditorActivity(cm, clear) + } + + function fadeIn(tooltip) { + document.body.appendChild(tooltip); + tooltip.style.opacity = "1"; + } + + function fadeOut(tooltip) { + tooltip.style.opacity = "0"; + setTimeout(function() { remove(tooltip); }, 100); + } + + function makeTooltip(x, y, content, element) { + var node = element || elt("div", "kotlin-error-tooltip", content); + node.style.left = x + "px"; + node.style.top = y + "px"; + return node; + } + + function elt(tagname, cls /*, ... elts*/) { + var e = document.createElement(tagname); + if (cls) e.className = cls; + for (var i = 2; i < arguments.length; ++i) { + var elt = arguments[i]; + if (typeof elt == "string") elt = document.createTextNode(elt); + e.appendChild(elt); + } + return e; + } + + function remove(node) { + var p = node && node.parentNode; + if (p) p.removeChild(node); + } + + function onEditorActivity(cm, f) { + cm.on("cursorActivity", f); + cm.on("blur", f); + cm.on("scroll", f); + cm.on("setDoc", f); + return function() { + cm.off("cursorActivity", f); + cm.off("blur", f); + cm.off("scroll", f); + cm.off("setDoc", f); + } + } + + Kernel.prototype.listErrors = function (code, callback) { + var callbacks; + if (callback) { + callbacks = { shell : { reply : callback } }; + } + var content = { + code : code + }; + return this.send_shell_message("list_errors_request", content, callbacks); + }; + + } + + return {onload: onload}; + +}); \ No newline at end of file diff --git a/resources/notebook-extension/wavyline-orange.gif b/resources/notebook-extension/wavyline-orange.gif new file mode 100644 index 000000000..69bf56b34 Binary files /dev/null and b/resources/notebook-extension/wavyline-orange.gif differ diff --git a/resources/notebook-extension/wavyline-red.gif b/resources/notebook-extension/wavyline-red.gif new file mode 100644 index 000000000..8766b3c8a Binary files /dev/null and b/resources/notebook-extension/wavyline-red.gif differ diff --git a/samples/Titanic.ipynb b/samples/Titanic.ipynb new file mode 100644 index 000000000..6c92f3968 --- /dev/null +++ b/samples/Titanic.ipynb @@ -0,0 +1,1449 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%use krangl-typed-local, khttp" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "// to see autogenerated code, uncomment the line below:\n", + "//%trackExecution -generated" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "val response = khttp.get(\"http://biostat.mc.vanderbilt.edu/wiki/pub/Main/DataSets/titanic.txt\")\n", + "val cleanedText = response.text.replace(\"\\\"Molly\\\"\", \"Molly\").replace(\"row.names\", \"row\").replace(\"home.dest\", \"home\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "// convert data to dataframe, generate marker interface for typed data frame wrapper and extension properties for it\n", + "val df = DataFrame.readDelim(cleanedText.byteInputStream())" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female
    131st1Aubert, Mrs Leontine PaulinenullCherbourgParis, FranceB-3517477 L69 6s9female
    141st1Barkworth, Mr Algernon H.nullSouthamptonHessle, YorksA-23Bmale
    151st0Baumann, Mr John D.nullSouthamptonNew York, NYmale
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Select" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "name [Str][1313]: Allen, Miss Elisabeth Walton, Allison, Miss Helen Loraine, Allison, Mr Hudson Josh..." + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// get typed column as extension property\n", + "df.name" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    name
    Allen, Miss Elisabeth Walton
    Allison, Miss Helen Loraine
    Allison, Mr Hudson Joshua Creighton
    Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)
    Allison, Master Hudson Trevor
    Anderson, Mr Harry
    Andrews, Miss Kornelia Theodosia
    Andrews, Mr Thomas, jr
    Appleton, Mrs Edward Dale (Charlotte Lamson)
    Artagaveytia, Mr Ramon
    Astor, Colonel John Jacob
    Astor, Mrs John Jacob (Madeleine Talmadge Force)
    Aubert, Mrs Leontine Pauline
    Barkworth, Mr Algernon H.
    Baumann, Mr John D.
    Baxter, Mrs James (Helene DeLaudeniere Chaput)
    Baxter, Mr Quigg Edmond
    Beattie, Mr Thomson
    Beckwith, Mr Richard Leonard
    Beckwith, Mrs Richard Leonard (Sallie Monypeny)

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// select single column -> returns DataFrame\n", + "df.select{name}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    nameageembarked
    Allen, Miss Elisabeth Walton29.0Southampton
    Allison, Miss Helen Loraine2.0Southampton
    Allison, Mr Hudson Joshua Creighton30.0Southampton
    Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0Southampton
    Allison, Master Hudson Trevor0.9167Southampton
    Anderson, Mr Harry47.0Southampton
    Andrews, Miss Kornelia Theodosia63.0Southampton
    Andrews, Mr Thomas, jr39.0Southampton
    Appleton, Mrs Edward Dale (Charlotte Lamson)58.0Southampton
    Artagaveytia, Mr Ramon71.0Cherbourg
    Astor, Colonel John Jacob47.0Cherbourg
    Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0Cherbourg
    Aubert, Mrs Leontine PaulinenullCherbourg
    Barkworth, Mr Algernon H.nullSouthampton
    Baumann, Mr John D.nullSouthampton
    Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0Cherbourg
    Baxter, Mr Quigg Edmond24.0Cherbourg
    Beattie, Mr Thomson36.0Cherbourg
    Beckwith, Mr Richard Leonard37.0Southampton
    Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0Southampton

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// select several columns\n", + "df.select{columns(name, age, embarked)}" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    nameagesex
    Allen, Miss Elisabeth Walton29.0female
    Allison, Miss Helen Loraine2.0female
    Allison, Mr Hudson Joshua Creighton30.0male
    Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0female
    Allison, Master Hudson Trevor0.9167male
    Anderson, Mr Harry47.0male
    Andrews, Miss Kornelia Theodosia63.0female
    Andrews, Mr Thomas, jr39.0male
    Appleton, Mrs Edward Dale (Charlotte Lamson)58.0female
    Artagaveytia, Mr Ramon71.0male
    Astor, Colonel John Jacob47.0male
    Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0female
    Aubert, Mrs Leontine Paulinenullfemale
    Barkworth, Mr Algernon H.nullmale
    Baumann, Mr John D.nullmale
    Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0female
    Baxter, Mr Quigg Edmond24.0male
    Beattie, Mr Thomson36.0male
    Beckwith, Mr Richard Leonard37.0male
    Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// another way to select columns without compile-time check\n", + "df.select(df.name, df.age, df.sex)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    pclassnameembarkedhomeroomticketboatsex
    1stAllen, Miss Elisabeth WaltonSouthamptonSt Louis, MOB-524160 L2212female
    1stAllison, Miss Helen LoraineSouthamptonMontreal, PQ / Chesterville, ONC26female
    1stAllison, Mr Hudson Joshua CreightonSouthamptonMontreal, PQ / Chesterville, ONC26(135)male
    1stAllison, Mrs Hudson J.C. (Bessie Waldo Daniels)SouthamptonMontreal, PQ / Chesterville, ONC26female
    1stAllison, Master Hudson TrevorSouthamptonMontreal, PQ / Chesterville, ONC2211male
    1stAnderson, Mr HarrySouthamptonNew York, NYE-123male
    1stAndrews, Miss Kornelia TheodosiaSouthamptonHudson, NYD-713502 L7710female
    1stAndrews, Mr Thomas, jrSouthamptonBelfast, NIA-36male
    1stAppleton, Mrs Edward Dale (Charlotte Lamson)SouthamptonBayside, Queens, NYC-1012female
    1stArtagaveytia, Mr RamonCherbourgMontevideo, Uruguay(22)male
    1stAstor, Colonel John JacobCherbourgNew York, NY17754 L224 10s 6d(124)male
    1stAstor, Mrs John Jacob (Madeleine Talmadge Force)CherbourgNew York, NY17754 L224 10s 6d4female
    1stAubert, Mrs Leontine PaulineCherbourgParis, FranceB-3517477 L69 6s9female
    1stBarkworth, Mr Algernon H.SouthamptonHessle, YorksA-23Bmale
    1stBaumann, Mr John D.SouthamptonNew York, NYmale
    1stBaxter, Mrs James (Helene DeLaudeniere Chaput)CherbourgMontreal, PQB-58/606female
    1stBaxter, Mr Quigg EdmondCherbourgMontreal, PQB-58/60male
    1stBeattie, Mr ThomsonCherbourgWinnipeg, MNC-6male
    1stBeckwith, Mr Richard LeonardSouthamptonNew York, NYD-355male
    1stBeckwith, Mrs Richard Leonard (Sallie Monypeny)SouthamptonNew York, NYD-355female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// select columns filtered by predicate\n", + "df.selectIf{valueClass == String::class}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Index" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Row indexing\n", + "df[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Allen, Miss Elisabeth Walton" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Column indexing\n", + "df.name[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Allen, Miss Elisabeth Walton" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Same result\n", + "df[0].name" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Filter" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female
    131st1Aubert, Mrs Leontine PaulinenullCherbourgParis, FranceB-3517477 L69 6s9female
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female
    241st1Bishop, Mrs Dickinson H. (Helen Walton)19.0CherbourgDowagiac, MIB-497female
    281st1Bonnell, Miss Caroline30.0SouthamptonYoungstown, OHC-78female
    291st1Bonnell, Miss Elizabeth58.0SouthamptonBirkdale, England Cleveland, OhioC-1038female
    311st1Bowen, Miss Grace Scott45.0CherbourgCooperstown, NY4female
    321st1Bowerman, Miss Elsie Edith22.0SouthamptonSt Leonards-on-Sea, England Ohio6female
    371st1Brown, Mrs James Joseph (Margaret Molly Tobin)44.0CherbourgDenver, CO17610 L27 15s 5d6female
    381st1Brown, Mrs John Murray (Caroline Lane Lamson)59.0SouthamptonBelmont, MAC-101Dfemale
    391st1Bucknell, Mrs William Robert (Emma Eliza Ward)60.0CherbourgPhiladelphia, PA8female
    421st1Candee, Mrs Edward (Helen Churchill Hungerford)53.0CherbourgWashington, DC6female
    431st1Cardeza, Mrs James Warburton Martinez (Charlott...58.0CherbourgGermantown, Philadelphia, PAB-51/3/517755 L512 6s3female
    491st1Carter, Mrs William Ernest (Lucile Polk)36.0SouthamptonBryn Mawr, PA4female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// filter rows by predicate. Predicate receiver is of type TypedDataFrameRow<*> with generated extension properties\n", + "df.filter {sex == \"female\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Operator call corresponds to a dot-qualified call 'age.compareTo(50)' which is not allowed on a nullable receiver 'age'." + ] + } + ], + "source": [ + "df.filter { age > 50 } // compilation error, because 'age' is a nullable property" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female
    211st1Behr, Mr Karl Howell26.0CherbourgNew York, NYC-1485male
    221st0Birnbaum, Mr Jakob25.0CherbourgSan Francisco, CA(148)male
    231st1Bishop, Mr Dickinson H.25.0CherbourgDowagiac, MIB-497male

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// filter rows where 'age' is not null. \n", + "val withAges = df.filterNotNull {age}\n", + "withAges" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    291st1Bonnell, Miss Elizabeth58.0SouthamptonBirkdale, England Cleveland, OhioC-1038female
    381st1Brown, Mrs John Murray (Caroline Lane Lamson)59.0SouthamptonBelmont, MAC-101Dfemale
    391st1Bucknell, Mrs William Robert (Emma Eliza Ward)60.0CherbourgPhiladelphia, PA8female
    421st1Candee, Mrs Edward (Helen Churchill Hungerford)53.0CherbourgWashington, DC6female
    431st1Cardeza, Mrs James Warburton Martinez (Charlott...58.0CherbourgGermantown, Philadelphia, PAB-51/3/517755 L512 6s3female
    681st1Compton, Mrs Alexander Taylor (Mary Eliza Inger...64.0CherbourgLakewood, NJ14female
    711st1Cornell, Mrs Robert Clifford (Malvina Helen Lam...55.0SouthamptonNew York, NYC-1012female
    731st0Crosby, Captain Edward Gifford70.0SouthamptonMilwaukee, WI(269)male
    741st1Crosby, Mrs Edward Gifford (Catherine Elizabeth...69.0SouthamptonMilwaukee, WI5female
    951st1Eustis, Miss Elizabeth Mussey53.0CherbourgBrookline, MA4female
    1041st0Fortune, Mr Mark64.0SouthamptonWinnipeg, MBmale
    1051st1Fortune, Mrs Mark (Mary McDougald)60.0SouthamptonWinnipeg, MB10female
    1111st1Frolicher-Stehli, Mr Maxmillian60.0CherbourgZurich, Switzerland5male
    1201st0Goldschmidt, Mr George B.71.0CherbourgNew York, NYmale
    1211st1Gracie, Colonel Archibald IV54.0SouthamptonWashington, DCC-51113780 L28 10sBmale
    1241st1Graham, Mrs William Thompson (Edith Junkins)58.0SouthamptonGreenwich, CTC-9117582 L153 9s 3d3female
    1351st0Hays, Mr Charles Melville55.0SouthamptonMontreal, PQ(307)male

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// now filtration works\n", + "withAges.filter {age > 50}" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    741st1Crosby, Mrs Edward Gifford (Catherine Elizabeth...69.0SouthamptonMilwaukee, WI5female
    " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// find the oldest survived woman\n", + "withAges.filter {survived == 1 && sex == \"female\"}.maxBy{age}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sort" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    5062nd0Mitchell, Mr Henry Michael71.0SouthamptonGuernsey / Montclair, NJ and/or Toledo, Ohiomale
    1201st0Goldschmidt, Mr George B.71.0CherbourgNew York, NYmale
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    731st0Crosby, Captain Edward Gifford70.0SouthamptonMilwaukee, WI(269)male
    741st1Crosby, Mrs Edward Gifford (Catherine Elizabeth...69.0SouthamptonMilwaukee, WI5female
    2531st0Straus, Mr Isidor67.0SouthamptonNew York, NY17483 L221 15s 7d(96)male
    7733rd0Dewan, Mr Frank65.0Queenstownmale
    1801st0Millet, Mr Francis Davis65.0SouthamptonEast Bridgewater, MA(249)male
    5102nd0Myles, Mr Thomas Francis64.0QueenstownCambridge, MAmale
    2711st0Warren, Mr Frank Manley64.0CherbourgPortland, ORmale
    1941st0Ostby, Mr Engelhart Cornelius64.0CherbourgProvidence, RI(234)male
    1921st0Nicholson, Mr Arthur Ernest64.0SouthamptonIsle of Wight, England(263)male
    1041st0Fortune, Mr Mark64.0SouthamptonWinnipeg, MBmale
    681st1Compton, Mrs Alexander Taylor (Mary Eliza Inger...64.0CherbourgLakewood, NJ14female
    2541st0Straus, Mrs Isidor (Ida Blun)63.0SouthamptonNew York, NY17483 L221 15s 7dfemale
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female
    2521st1Stone, Mrs George Nelson (Martha E.)62.0Cincinatti, OH6female
    2471st0Stead, Mr William Thomas62.0SouthamptonWimbledon Park, London / Hayling Island, HantsC-89male
    2691st0Van Derhoef, Mr Wyckoff61.0SouthamptonBrooklyn, NY(245)male
    2551st0Sutton, Mr Frederick61.0SouthamptonHaddenfield, NJ(46)male

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// sort by single column\n", + "withAges.sortedByDesc {age}" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    7643rd1Dean, Miss Elizabeth Gladys (Millvena)0.1667SouthamptonDevon, England Wichita, KS12female
    7523rd0Danbom, Master Gilbert Sigvard Emanuel0.3333SouthamptonStanton, IAmale
    6173rd1Aks, Master Philip0.8333SouthamptonLondon, England Norfolk, VA39209111male
    3592nd1Caldwell, Master Alden Gates0.8333SouthamptonBangkok, Thailand / Roseville, IL13male
    5452nd1Richards, Master George Sidney0.8333SouthamptonCornwall / Akron, OH4male
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male
    3402nd1Becker, Master Richard F.1.0SouthamptonGuntur, India / Benton Harbour, MI230136 L3911male
    7633rd1Dean, Master Bertram Vere1.0SouthamptonDevon, England Wichita, KS12male
    4262nd1Hamalainen, Master Viljo1.0SouthamptonDetroit, MImale
    4792nd1LaRoche, Miss Louise1.0CherbourgParis / Haitifemale
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    6283rd0Andersson, Miss Ellis Anna Maria2.0SouthamptonSweden Winnipeg, MNfemale
    4922nd1Mallet, Master Andre2.0CherbourgParis / Montreal, PQmale
    5132nd1Navratil, Master Edmond Roger2.0SouthamptonNice, France230080 L26Dmale
    5382nd1Quick, Miss Phyllis May2.0SouthamptonPlymouth, Devon / Detroit, MI11female
    5862nd1Wells, Master Ralph Lester2.0SouthamptonCornwall / Akron, OHmale
    6453rd1Aspland, Master Edvin Rojj Felix3.0SouthamptonSweden Worcester, MA4male
    6583rd1Baclini, Miss Eugenie3.0CherbourgSyria New York, NYfemale
    7383rd1Coutts, Master Neville3.0SouthamptonEngland Brooklyn, NY2male
    4802nd1LaRoche, Miss Simonne3.0CherbourgParis / Haitifemale

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// sort by several columns\n", + "withAges.sortedBy {columns(age, name)}" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    7643rd1Dean, Miss Elizabeth Gladys (Millvena)0.1667SouthamptonDevon, England Wichita, KS12female
    7523rd0Danbom, Master Gilbert Sigvard Emanuel0.3333SouthamptonStanton, IAmale
    6173rd1Aks, Master Philip0.8333SouthamptonLondon, England Norfolk, VA39209111male
    3592nd1Caldwell, Master Alden Gates0.8333SouthamptonBangkok, Thailand / Roseville, IL13male
    5452nd1Richards, Master George Sidney0.8333SouthamptonCornwall / Akron, OH4male
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male
    3402nd1Becker, Master Richard F.1.0SouthamptonGuntur, India / Benton Harbour, MI230136 L3911male
    7633rd1Dean, Master Bertram Vere1.0SouthamptonDevon, England Wichita, KS12male
    4262nd1Hamalainen, Master Viljo1.0SouthamptonDetroit, MImale
    4792nd1LaRoche, Miss Louise1.0CherbourgParis / Haitifemale
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    6283rd0Andersson, Miss Ellis Anna Maria2.0SouthamptonSweden Winnipeg, MNfemale
    4922nd1Mallet, Master Andre2.0CherbourgParis / Montreal, PQmale
    5132nd1Navratil, Master Edmond Roger2.0SouthamptonNice, France230080 L26Dmale
    5382nd1Quick, Miss Phyllis May2.0SouthamptonPlymouth, Devon / Detroit, MI11female
    5862nd1Wells, Master Ralph Lester2.0SouthamptonCornwall / Akron, OHmale
    6453rd1Aspland, Master Edvin Rojj Felix3.0SouthamptonSweden Worcester, MA4male
    6583rd1Baclini, Miss Eugenie3.0CherbourgSyria New York, NYfemale
    7383rd1Coutts, Master Neville3.0SouthamptonEngland Brooklyn, NY2male
    4802nd1LaRoche, Miss Simonne3.0CherbourgParis / Haitifemale

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// another way\n", + "withAges.sortedBy(withAges.age, withAges.name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Add Columns" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsexyear
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female1883.0
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female1910.0
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male1882.0
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female1887.0
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male1911.0833
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male1865.0
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female1849.0
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male1873.0
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female1854.0
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male1841.0
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male1865.0
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female1893.0
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female1862.0
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male1888.0
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male1876.0
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male1875.0
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female1865.0
    211st1Behr, Mr Karl Howell26.0CherbourgNew York, NYC-1485male1886.0
    221st0Birnbaum, Mr Jakob25.0CherbourgSan Francisco, CA(148)male1887.0
    231st1Bishop, Mr Dickinson H.25.0CherbourgDowagiac, MIB-497male1887.0

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// add new column and store result in a new field\n", + "val withYear = withAges.add(\"year\") {1912 - age}\n", + "withYear" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "year [Dbl][633]: 1883, 1910, 1882, 1887, 1911.083, 1865, 1849, 1873, 1854, 1841, 1865, 1893, 1862, 1..." + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// check new column\n", + "withYear.year" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsexyeardied
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female1883.0false
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female1910.0true
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male1882.0true
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female1887.0true
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male1911.0833false
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male1865.0false
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female1849.0false
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male1873.0true
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female1854.0false
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male1841.0true
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male1865.0true
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female1893.0false
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female1862.0false
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male1888.0true
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male1876.0true
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male1875.0false
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female1865.0false
    211st1Behr, Mr Karl Howell26.0CherbourgNew York, NYC-1485male1886.0false
    221st0Birnbaum, Mr Jakob25.0CherbourgSan Francisco, CA(148)male1887.0true
    231st1Bishop, Mr Dickinson H.25.0CherbourgDowagiac, MIB-497male1887.0false

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// add several columns\n", + "withAges.add {\n", + " \"year\" {1912-age}\n", + " \"died\" {survived == 0}\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsexyeardied
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female1883.0false
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female1910.0true
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male1882.0true
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female1887.0true
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male1911.0833false
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male1865.0false
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female1849.0false
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male1873.0true
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female1854.0false
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male1841.0true
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male1865.0true
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female1893.0false
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female1862.0false
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male1888.0true
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male1876.0true
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male1875.0false
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female1865.0false
    211st1Behr, Mr Karl Howell26.0CherbourgNew York, NYC-1485male1886.0false
    221st0Birnbaum, Mr Jakob25.0CherbourgSan Francisco, CA(148)male1887.0true
    231st1Bishop, Mr Dickinson H.25.0CherbourgDowagiac, MIB-497male1887.0false

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// plus is overloaded for adding columns\n", + "withAges + {\n", + " \"year\" {1912-age}\n", + " \"died\" {survived == 0}\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "// another way to build new column via column arithmetics\n", + "val birthYear = withAges.age * (-1) + 1912" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsexyear
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female1883.0
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female1910.0
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male1882.0
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female1887.0
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male1911.0833
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male1865.0
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female1849.0
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male1873.0
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female1854.0
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male1841.0
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male1865.0
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female1893.0
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female1862.0
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male1888.0
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male1876.0
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male1875.0
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female1865.0
    211st1Behr, Mr Karl Howell26.0CherbourgNew York, NYC-1485male1886.0
    221st0Birnbaum, Mr Jakob25.0CherbourgSan Francisco, CA(148)male1887.0
    231st1Bishop, Mr Dickinson H.25.0CherbourgDowagiac, MIB-497male1887.0

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// new column can be added to dataframe with '+' operator\n", + "withAges + birthYear.rename(\"year\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsexrow duplicatepclass duplicatesurvived duplicatename duplicateage duplicateembarked duplicatehome duplicateroom duplicateticket duplicateboat duplicatesex duplicate
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female
    211st1Behr, Mr Karl Howell26.0CherbourgNew York, NYC-1485male211st1Behr, Mr Karl Howell26.0CherbourgNew York, NYC-1485male
    221st0Birnbaum, Mr Jakob25.0CherbourgSan Francisco, CA(148)male221st0Birnbaum, Mr Jakob25.0CherbourgSan Francisco, CA(148)male
    231st1Bishop, Mr Dickinson H.25.0CherbourgDowagiac, MIB-497male231st1Bishop, Mr Dickinson H.25.0CherbourgDowagiac, MIB-497male

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Iterable of columns can also be added with '+' \n", + "withAges + withAges.columns.map {it.rename(it.name + \" duplicate\")}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Remove columns" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomboatsex
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-52female
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-710female
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY(124)male
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY4female
    131st1Aubert, Mrs Leontine PaulinenullCherbourgParis, FranceB-359female
    141st1Barkworth, Mr Algernon H.nullSouthamptonHessle, YorksA-23Bmale
    151st0Baumann, Mr John D.nullSouthamptonNew York, NYmale
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// remove single column\n", + "df.remove{ticket}" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    nameageembarkedhomeboatsex
    Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MO2female
    Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONfemale
    Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ON(135)male
    Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONfemale
    Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ON11male
    Anderson, Mr Harry47.0SouthamptonNew York, NY3male
    Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NY10female
    Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NImale
    Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NY2female
    Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    Astor, Colonel John Jacob47.0CherbourgNew York, NY(124)male
    Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY4female
    Aubert, Mrs Leontine PaulinenullCherbourgParis, France9female
    Barkworth, Mr Algernon H.nullSouthamptonHessle, YorksBmale
    Baumann, Mr John D.nullSouthamptonNew York, NYmale
    Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQ6female
    Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQmale
    Beattie, Mr Thomson36.0CherbourgWinnipeg, MNmale
    Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NY5male
    Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NY5female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// remove several columns\n", + "df.remove {columns(row, pclass, ticket, room, survived)}" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    survivednameageembarkedhomeroomticketboatsex
    1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212female
    0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)male
    0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26female
    1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211male
    1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123male
    1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female
    0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36male
    1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female
    0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male
    1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female
    1Aubert, Mrs Leontine PaulinenullCherbourgParis, FranceB-3517477 L69 6s9female
    1Barkworth, Mr Algernon H.nullSouthamptonHessle, YorksA-23Bmale
    0Baumann, Mr John D.nullSouthamptonNew York, NYmale
    1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606female
    0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60male
    0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6male
    1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355male
    1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// remove several columns by column instances\n", + "df.remove(df.row, df.pclass)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    survivednameageembarkedhometicketboatsex
    1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MO24160 L2212female
    0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONfemale
    0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ON(135)male
    0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONfemale
    1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ON11male
    1Anderson, Mr Harry47.0SouthamptonNew York, NY3male
    1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NY13502 L7710female
    0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NImale
    1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NY2female
    0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)male
    1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4female
    1Aubert, Mrs Leontine PaulinenullCherbourgParis, France17477 L69 6s9female
    1Barkworth, Mr Algernon H.nullSouthamptonHessle, YorksBmale
    0Baumann, Mr John D.nullSouthamptonNew York, NYmale
    1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQ6female
    0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQmale
    0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNmale
    1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NY5male
    1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NY5female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// '-' operator can also be used for removing columns\n", + "df - {row} - {pclass} - {room}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Group" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    embarkedn
    Southampton573
    Cherbourg203
    492
    Queenstown45
    " + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// group by single column\n", + "df.groupBy{ embarked }.count()" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    sexsurvivedn
    female1307
    female0156
    male0708
    male1142
    " + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// group by several columns\n", + "df.groupBy{ columns(sex, survived) }.count()" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    sexsurvivedn
    female1307
    female0156
    male0708
    male1142
    " + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// another way\n", + "df.groupBy(df.sex, df.survived).count()" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    embarkedtotal countsurvival rateaverage agemedian ageyoungestyoungest ageoldestoldest age
    Southampton46339.74082073434125629.85727127429805719.0Dean, Miss Elizabeth Gladys (Millvena)0.1667Mitchell, Mr Henry Michael71.0
    Cherbourg13360.90225563909774635.601503759398522.0LaRoche, Miss Louise1.0Artagaveytia, Mr Ramon71.0
    Queenstown3138.7096774193548429.03225806451612837.0Carr, Miss Helen16.0Dewan, Mr Frank65.0
    666.6666666666666647.83333333333333647.5Ovies y Rodriguez, Mr Servando28.0Stone, Mrs George Nelson (Martha E.)62.0
    " + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Various summarization operations on grouped data frame\n", + "withAges.groupBy{ embarked }.summarize {\n", + " \n", + " \"total count\" { size } // lamba expressions are computed for every group. Type of receiver: TypedDataFrame<*>\n", + " \"survival rate\" { count { survived == 1 }.toDouble() / size * 100 }\n", + " \n", + " \"average age\" { age.mean() } // column operations are also supported\n", + " \"median age\" { age.median() }\n", + " \n", + " val youngest = find { minBy {age}!! } // 'find' builds data frame, collecting one row for every group\n", + " \"youngest\" (youngest.name) // columns of collected data frame are passed in round parenthesis '()'\n", + " \"youngest age\" (youngest.age)\n", + " \n", + " val oldest = find { maxBy {age}!! }\n", + " \"oldest\" (oldest.name)\n", + " \"oldest age\" (oldest.age)\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Misc" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1313" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.size" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "68" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "withAges.count {age > 50 }" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    7643rd1Dean, Miss Elizabeth Gladys (Millvena)0.1667SouthamptonDevon, England Wichita, KS12female
    7523rd0Danbom, Master Gilbert Sigvard Emanuel0.3333SouthamptonStanton, IAmale
    3592nd1Caldwell, Master Alden Gates0.8333SouthamptonBangkok, Thailand / Roseville, IL13male
    5452nd1Richards, Master George Sidney0.8333SouthamptonCornwall / Akron, OH4male
    6173rd1Aks, Master Philip0.8333SouthamptonLondon, England Norfolk, VA39209111male
    " + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "withAges.sortedBy{age}.take(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    741st1Crosby, Mrs Edward Gifford (Catherine Elizabeth...69.0SouthamptonMilwaukee, WI5female
    731st0Crosby, Captain Edward Gifford70.0SouthamptonMilwaukee, WI(269)male
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    1201st0Goldschmidt, Mr George B.71.0CherbourgNew York, NYmale
    5062nd0Mitchell, Mr Henry Michael71.0SouthamptonGuernsey / Montclair, NJ and/or Toledo, Ohiomale
    " + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "withAges.sortedBy{age}.takeLast(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List <-> DataFrame conversion" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Allen, Miss Elisabeth Walton, Allison, Miss Helen Loraine, Allison, Mr Hudson Joshua Creighton, Allison, Mrs Hudson J.C. (Bessie Waldo Daniels), Allison, Master Hudson Trevor]" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// 'rows' field is Iterable> so it can be used in any stdlib extensions for Iterable\n", + "df.rows.map {it.name}.take(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    firstsecond
    12.0
    23.0
    34.0
    " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Sample List\n", + "data class Item(val first: Int, val second: Double)\n", + "val itemsList = listOf(Item(1,2.0), Item(2, 3.0), Item(3, 4.0))\n", + "\n", + "// List -> DataFrame by reflection\n", + "itemsList.toDataFrame()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    abc
    12.02.0
    23.06.0
    34.012.0
    " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// List -> DataFrame by mappings\n", + "itemsList.toDataFrame {\n", + " \"a\" {first}\n", + " \"b\" {second}\n", + " \"c\" {first*second}\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "// Convert data frame to a list of data class items\n", + "val passengers = df.toList(\"Passenger\")" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "class Line_139_jupyter$Passenger" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Check type of the element\n", + "passengers[0].javaClass" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Passenger(row=10, pclass=1st, survived=0, name=Artagaveytia, Mr Ramon, age=71.0, embarked=Cherbourg, home=Montevideo, Uruguay, room=, ticket=, boat=(22), sex=male)" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// Do any list operations\n", + "passengers.maxBy {it.age ?: .0}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Column-specific extensions for TypedDataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "// create marker interface to write column-specific extensions for data frame\n", + "@DataFrameType\n", + "interface SimplePerson {\n", + " val name: String\n", + " val age: Double\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "// create extension for any data frame with fields 'name' and 'age'\n", + "fun TypedDataFrame.getOlderThan(minAge: Double) = filter {age > minAge}" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsex
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710female
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012female
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)male
    291st1Bonnell, Miss Elizabeth58.0SouthamptonBirkdale, England Cleveland, OhioC-1038female
    381st1Brown, Mrs John Murray (Caroline Lane Lamson)59.0SouthamptonBelmont, MAC-101Dfemale
    391st1Bucknell, Mrs William Robert (Emma Eliza Ward)60.0CherbourgPhiladelphia, PA8female
    421st1Candee, Mrs Edward (Helen Churchill Hungerford)53.0CherbourgWashington, DC6female
    431st1Cardeza, Mrs James Warburton Martinez (Charlott...58.0CherbourgGermantown, Philadelphia, PAB-51/3/517755 L512 6s3female
    681st1Compton, Mrs Alexander Taylor (Mary Eliza Inger...64.0CherbourgLakewood, NJ14female
    711st1Cornell, Mrs Robert Clifford (Malvina Helen Lam...55.0SouthamptonNew York, NYC-1012female
    731st0Crosby, Captain Edward Gifford70.0SouthamptonMilwaukee, WI(269)male
    741st1Crosby, Mrs Edward Gifford (Catherine Elizabeth...69.0SouthamptonMilwaukee, WI5female
    951st1Eustis, Miss Elizabeth Mussey53.0CherbourgBrookline, MA4female
    1041st0Fortune, Mr Mark64.0SouthamptonWinnipeg, MBmale
    1051st1Fortune, Mrs Mark (Mary McDougald)60.0SouthamptonWinnipeg, MB10female
    1111st1Frolicher-Stehli, Mr Maxmillian60.0CherbourgZurich, Switzerland5male
    1201st0Goldschmidt, Mr George B.71.0CherbourgNew York, NYmale
    1211st1Gracie, Colonel Archibald IV54.0SouthamptonWashington, DCC-51113780 L28 10sBmale
    1241st1Graham, Mrs William Thompson (Edith Junkins)58.0SouthamptonGreenwich, CTC-9117582 L153 9s 3d3female
    1351st0Hays, Mr Charles Melville55.0SouthamptonMontreal, PQ(307)male

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// extension works even for objects that were created before marker interface declaration\n", + "withAges.getOlderThan(50.0)" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "@DataFrameType(isOpen = true)\n", + "interface Person{\n", + "\tval name: String\n", + "\tval age: Double\n", + "\tval home: String\n", + "\tval sex: String\n", + "}" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// code for marker interface can be auto-generated\n", + "// 'getScheme' method returns generated code without execution\n", + "withAges.select{columns(name,age,home,sex)}.generateInterface(\"Person\")" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "// 'extractScheme' method generates and executes code\n", + "withAges.select{columns(name,age,home,sex)}.extractInterface(\"Person\")" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [], + "source": [ + "// Now interface 'Person' is available, so we can write an extension method, \n", + "// that will work for any data frame with these four columns\n", + "fun TypedDataFrame.addSummary() = add(\"summary\"){\"$sex $name $age y.o. from $home\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    rowpclasssurvivednameageembarkedhomeroomticketboatsexsummary
    11st1Allen, Miss Elisabeth Walton29.0SouthamptonSt Louis, MOB-524160 L2212femalefemale Allen, Miss Elisabeth Walton 29.0 y.o. f...
    21st0Allison, Miss Helen Loraine2.0SouthamptonMontreal, PQ / Chesterville, ONC26femalefemale Allison, Miss Helen Loraine 2.0 y.o. fro...
    31st0Allison, Mr Hudson Joshua Creighton30.0SouthamptonMontreal, PQ / Chesterville, ONC26(135)malemale Allison, Mr Hudson Joshua Creighton 30.0 y...
    41st0Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)25.0SouthamptonMontreal, PQ / Chesterville, ONC26femalefemale Allison, Mrs Hudson J.C. (Bessie Waldo D...
    51st1Allison, Master Hudson Trevor0.9167SouthamptonMontreal, PQ / Chesterville, ONC2211malemale Allison, Master Hudson Trevor 0.9167 y.o. ...
    61st1Anderson, Mr Harry47.0SouthamptonNew York, NYE-123malemale Anderson, Mr Harry 47.0 y.o. from New York...
    71st1Andrews, Miss Kornelia Theodosia63.0SouthamptonHudson, NYD-713502 L7710femalefemale Andrews, Miss Kornelia Theodosia 63.0 y....
    81st0Andrews, Mr Thomas, jr39.0SouthamptonBelfast, NIA-36malemale Andrews, Mr Thomas, jr 39.0 y.o. from Belf...
    91st1Appleton, Mrs Edward Dale (Charlotte Lamson)58.0SouthamptonBayside, Queens, NYC-1012femalefemale Appleton, Mrs Edward Dale (Charlotte Lam...
    101st0Artagaveytia, Mr Ramon71.0CherbourgMontevideo, Uruguay(22)malemale Artagaveytia, Mr Ramon 71.0 y.o. from Mont...
    111st0Astor, Colonel John Jacob47.0CherbourgNew York, NY17754 L224 10s 6d(124)malemale Astor, Colonel John Jacob 47.0 y.o. from N...
    121st1Astor, Mrs John Jacob (Madeleine Talmadge Force)19.0CherbourgNew York, NY17754 L224 10s 6d4femalefemale Astor, Mrs John Jacob (Madeleine Talmadg...
    161st1Baxter, Mrs James (Helene DeLaudeniere Chaput)50.0CherbourgMontreal, PQB-58/606femalefemale Baxter, Mrs James (Helene DeLaudeniere C...
    171st0Baxter, Mr Quigg Edmond24.0CherbourgMontreal, PQB-58/60malemale Baxter, Mr Quigg Edmond 24.0 y.o. from Mon...
    181st0Beattie, Mr Thomson36.0CherbourgWinnipeg, MNC-6malemale Beattie, Mr Thomson 36.0 y.o. from Winnipe...
    191st1Beckwith, Mr Richard Leonard37.0SouthamptonNew York, NYD-355malemale Beckwith, Mr Richard Leonard 37.0 y.o. fro...
    201st1Beckwith, Mrs Richard Leonard (Sallie Monypeny)47.0SouthamptonNew York, NYD-355femalefemale Beckwith, Mrs Richard Leonard (Sallie Mo...
    211st1Behr, Mr Karl Howell26.0CherbourgNew York, NYC-1485malemale Behr, Mr Karl Howell 26.0 y.o. from New Yo...
    221st0Birnbaum, Mr Jakob25.0CherbourgSan Francisco, CA(148)malemale Birnbaum, Mr Jakob 25.0 y.o. from San Fran...
    231st1Bishop, Mr Dickinson H.25.0CherbourgDowagiac, MIB-497malemale Bishop, Mr Dickinson H. 25.0 y.o. from Dow...

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// for example, it works for 'withAges' data frame\n", + "withAges.addSummary()" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "// data frame can also be converted to a list of objects implementing 'Person' interface that was generated above\n", + "val persons = withAges.toList()" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "class Line_176_jupyter$PersonImpl" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// check element type\n", + "persons[0].javaClass" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[PersonImpl(age=29.0, home=St Louis, MO, name=Allen, Miss Elisabeth Walton, sex=female), PersonImpl(age=2.0, home=Montreal, PQ / Chesterville, ON, name=Allison, Miss Helen Loraine, sex=female), PersonImpl(age=30.0, home=Montreal, PQ / Chesterville, ON, name=Allison, Mr Hudson Joshua Creighton, sex=male), PersonImpl(age=25.0, home=Montreal, PQ / Chesterville, ON, name=Allison, Mrs Hudson J.C. (Bessie Waldo Daniels), sex=female), PersonImpl(age=0.9167, home=Montreal, PQ / Chesterville, ON, name=Allison, Master Hudson Trevor, sex=male), PersonImpl(age=47.0, home=New York, NY, name=Anderson, Mr Harry, sex=male), PersonImpl(age=63.0, home=Hudson, NY, name=Andrews, Miss Kornelia Theodosia, sex=female), PersonImpl(age=39.0, home=Belfast, NI, name=Andrews, Mr Thomas, jr, sex=male), PersonImpl(age=58.0, home=Bayside, Queens, NY, name=Appleton, Mrs Edward Dale (Charlotte Lamson), sex=female), PersonImpl(age=71.0, home=Montevideo, Uruguay, name=Artagaveytia, Mr Ramon, sex=male), PersonImpl(age=47.0, home=New York, NY, name=Astor, Colonel John Jacob, sex=male), PersonImpl(age=19.0, home=New York, NY, name=Astor, Mrs John Jacob (Madeleine Talmadge Force), sex=female), PersonImpl(age=50.0, home=Montreal, PQ, name=Baxter, Mrs James (Helene DeLaudeniere Chaput), sex=female), PersonImpl(age=24.0, home=Montreal, PQ, name=Baxter, Mr Quigg Edmond, sex=male), PersonImpl(age=36.0, home=Winnipeg, MN, name=Beattie, Mr Thomson, sex=male), PersonImpl(age=37.0, home=New York, NY, name=Beckwith, Mr Richard Leonard, sex=male), PersonImpl(age=47.0, home=New York, NY, name=Beckwith, Mrs Richard Leonard (Sallie Monypeny), sex=female), PersonImpl(age=26.0, home=New York, NY, name=Behr, Mr Karl Howell, sex=male), PersonImpl(age=25.0, home=San Francisco, CA, name=Birnbaum, Mr Jakob, sex=male), PersonImpl(age=25.0, home=Dowagiac, MI, name=Bishop, Mr Dickinson H., sex=male), PersonImpl(age=19.0, home=Dowagiac, MI, name=Bishop, Mrs Dickinson H. (Helen Walton), sex=female), PersonImpl(age=28.0, home=Stockholm, Sweden / Washington, DC, name=Bjornstrm-Steffansson, Mr Mauritz Hakan, sex=male), PersonImpl(age=45.0, home=Trenton, NJ, name=Blackwell, Mr Stephen Weart, sex=male), PersonImpl(age=39.0, home=Glen Ridge, NJ, name=Blank, Mr Henry, sex=male), PersonImpl(age=30.0, home=Youngstown, OH, name=Bonnell, Miss Caroline, sex=female), PersonImpl(age=58.0, home=Birkdale, England Cleveland, Ohio, name=Bonnell, Miss Elizabeth, sex=female), PersonImpl(age=45.0, home=Cooperstown, NY, name=Bowen, Miss Grace Scott, sex=female), PersonImpl(age=22.0, home=St Leonards-on-Sea, England Ohio, name=Bowerman, Miss Elsie Edith, sex=female), PersonImpl(age=41.0, home=Pomeroy, WA, name=Brady, Mr John Bertram, sex=male), PersonImpl(age=48.0, home=Omaha, NE, name=Brandeis, Mr Emil, sex=male), PersonImpl(age=44.0, home=Denver, CO, name=Brown, Mrs James Joseph (Margaret Molly Tobin), sex=female), PersonImpl(age=59.0, home=Belmont, MA, name=Brown, Mrs John Murray (Caroline Lane Lamson), sex=female), PersonImpl(age=60.0, home=Philadelphia, PA, name=Bucknell, Mrs William Robert (Emma Eliza Ward), sex=female), PersonImpl(age=45.0, home=Washington, DC, name=Butt, Major Archibald Willingham, sex=male), PersonImpl(age=53.0, home=Washington, DC, name=Candee, Mrs Edward (Helen Churchill Hungerford), sex=female), PersonImpl(age=58.0, home=Germantown, Philadelphia, PA, name=Cardeza, Mrs James Warburton Martinez (Charlotte Wardle Drake), sex=female), PersonImpl(age=36.0, home=Austria-Hungary / Germantown, Philadelphia, PA, name=Cardeza, Mr Thomas Drake Martinez, sex=male), PersonImpl(age=33.0, home=New York, NY, name=Carlsson, Mr Frans Olof, sex=male), PersonImpl(age=36.0, home=Bryn Mawr, PA, name=Carter, Mr William Ernest, sex=male), PersonImpl(age=36.0, home=Bryn Mawr, PA, name=Carter, Mrs William Ernest (Lucile Polk), sex=female), PersonImpl(age=14.0, home=Bryn Mawr, PA, name=Carter, Miss Lucile Polk, sex=female), PersonImpl(age=11.0, home=Bryn Mawr, PA, name=Carter, Master William T. II, sex=male), PersonImpl(age=49.0, home=Ascot, Berkshire / Rochester, NY, name=Case, Mr Howard Brown, sex=male), PersonImpl(age=36.0, home=Little Onn Hall, Staffs, name=Cavendish, Mr Tyrell William, sex=male), PersonImpl(age=46.0, home=Amenia, ND, name=Chaffee, Mr Herbert Fuller, sex=male), PersonImpl(age=47.0, home=Amenia, ND, name=Chaffee, Mrs Herbert Fuller (Carrie Constance Toogood), sex=female), PersonImpl(age=27.0, home=New York, NY / Ithaca, NY, name=Chambers, Mr Norman Campbell, sex=male), PersonImpl(age=31.0, home=New York, NY / Ithaca, NY, name=Chambers, Mrs Norman Campbell (Bertha Griggs), sex=female), PersonImpl(age=27.0, home=Los Angeles, CA, name=Clark, Mr Walter Miller, sex=male), PersonImpl(age=26.0, home=Los Angeles, CA, name=Clark, Mrs Walter Miller (Virginia McDowell), sex=female), PersonImpl(age=64.0, home=Lakewood, NJ, name=Compton, Mrs Alexander Taylor (Mary Eliza Ingersoll), sex=female), PersonImpl(age=37.0, home=Lakewood, NJ, name=Compton, Mr Alexander Taylor, Jr, sex=male), PersonImpl(age=39.0, home=Lakewood, NJ, name=Compton, Miss Sara Rebecca, sex=female), PersonImpl(age=55.0, home=New York, NY, name=Cornell, Mrs Robert Clifford (Malvina Helen Lamson), sex=female), PersonImpl(age=70.0, home=Milwaukee, WI, name=Crosby, Captain Edward Gifford, sex=male), PersonImpl(age=69.0, home=Milwaukee, WI, name=Crosby, Mrs Edward Gifford (Catherine Elizabeth Halstead), sex=female), PersonImpl(age=36.0, home=Milwaukee, WI, name=Crosby, Miss Harriet R., sex=female), PersonImpl(age=39.0, home=New York, NY, name=Cumings, Mr John Bradley, sex=male), PersonImpl(age=38.0, home=New York, NY, name=Cumings, Mrs John Bradley (Florence Briggs Thayer), sex=female), PersonImpl(age=27.0, home=Philadelphia, PA, name=Daniel, Mr Robert Williams, sex=male), PersonImpl(age=31.0, home=Montreal, PQ, name=Davidson, Mr Thornton, sex=male), PersonImpl(age=27.0, home=Montreal, PQ, name=Davidson, Mrs Thornton (Orian Hays), sex=female), PersonImpl(age=31.0, home=Calgary, AB, name=Dick, Mr Albert Adrian, sex=male), PersonImpl(age=17.0, home=Calgary, AB, name=Dick, Mrs Albert Adrian Vera Gillespie, sex=female), PersonImpl(age=4.0, home=San Francisco, CA, name=Dodge, Master Washington, sex=male), PersonImpl(age=27.0, home=Montreal, PQ, name=Douglas, Mrs Frederick Charles (Suzette Baxter), sex=female), PersonImpl(age=50.0, home=Deephaven, MN / Cedar Rapids, IA, name=Douglas, Mr Walter Donald, sex=male), PersonImpl(age=48.0, home=Deephaven, MN / Cedar Rapids, IA, name=Douglas, Mrs Walter Donald (Mahala Dutton), sex=female), PersonImpl(age=49.0, home=London / Paris, name=Duff Gordon, Sir Cosmo Edmund, sex=male), PersonImpl(age=48.0, home=London / Paris, name=Duff Gordon, Lady (Lucille Wallace Sutherland), sex=female), PersonImpl(age=39.0, home=Philadelphia, PA, name=Dulles, Mr William Crothers, sex=male), PersonImpl(age=23.0, home=Mt Airy, Philadelphia, PA, name=Earnshaw, Mrs Boulton (Olive Potter), sex=female), PersonImpl(age=53.0, home=Brookline, MA, name=Eustis, Miss Elizabeth Mussey, sex=female), PersonImpl(age=36.0, home=New York, NY, name=Evans, Miss Edith Corse, sex=female), PersonImpl(age=30.0, home=New York, NY, name=Foreman, Mr Benjamin Laventall, sex=male), PersonImpl(age=24.0, home=Winnipeg, MB, name=Fortune, Miss Alice Elizabeth, sex=female), PersonImpl(age=19.0, home=Winnipeg, MB, name=Fortune, Mr Charles Alexander, sex=male), PersonImpl(age=28.0, home=Winnipeg, MB, name=Fortune, Miss Ethel Flora, sex=female), PersonImpl(age=23.0, home=Winnipeg, MB, name=Fortune, Miss Mabel, sex=female), PersonImpl(age=64.0, home=Winnipeg, MB, name=Fortune, Mr Mark, sex=male), PersonImpl(age=60.0, home=Winnipeg, MB, name=Fortune, Mrs Mark (Mary McDougald), sex=female), PersonImpl(age=49.0, home=New York, NY, name=Frauenthal, Dr Henry William, sex=male), PersonImpl(age=44.0, home=New York, NY, name=Frauenthal, Mr Isaac Gerald, sex=male), PersonImpl(age=22.0, home=Zurich, Switzerland, name=Frolicher, Miss Marguerite, sex=female), PersonImpl(age=60.0, home=Zurich, Switzerland, name=Frolicher-Stehli, Mr Maxmillian, sex=male), PersonImpl(age=48.0, home=Zurich, Switzerland, name=Frolicher-Stehli, Mrs Maxmillian (Margaretha Emerentia Stehli), sex=female), PersonImpl(age=37.0, home=Scituate, MA, name=Futrelle, Mr Jacques, sex=male), PersonImpl(age=35.0, home=Scituate, MA, name=Futrelle, Mrs Jacques (May Peel), sex=female), PersonImpl(age=47.0, home=St Anne's-on-Sea, Lancashire, name=Gee, Mr Arthur H., sex=male), PersonImpl(age=22.0, home=New York, NY, name=Gibson, Miss Dorothy, sex=female), PersonImpl(age=45.0, home=New York, NY, name=Gibson, Mrs Leonard (Pauline C. Boeson), sex=female), PersonImpl(age=49.0, home=Paris, France / New York, NY, name=Goldenberg, Mr Samuel L., sex=male), PersonImpl(age=71.0, home=New York, NY, name=Goldschmidt, Mr George B., sex=male), PersonImpl(age=54.0, home=Washington, DC, name=Gracie, Colonel Archibald IV, sex=male), PersonImpl(age=38.0, home=Winnipeg, MB, name=Graham, Mr George Edward, sex=male), PersonImpl(age=19.0, home=Greenwich, CT, name=Graham, Miss Margaret Edith, sex=female), PersonImpl(age=58.0, home=Greenwich, CT, name=Graham, Mrs William Thompson (Edith Junkins), sex=female), PersonImpl(age=45.0, home=New York, NY, name=Greenfield, Mrs Leo David (Blanche Strouse), sex=female), PersonImpl(age=23.0, home=New York, NY, name=Greenfield, Mr William Bertram, sex=male), PersonImpl(age=46.0, home=New York, NY, name=Guggenheim, Mr Benjamin, sex=male), PersonImpl(age=25.0, home=Brooklyn, NY, name=Harder, Mr George Achilles, sex=male), PersonImpl(age=21.0, home=Brooklyn, NY, name=Harder, Mrs George Achilles (Dorothy Annan), sex=female), PersonImpl(age=48.0, home=New York, NY, name=Harper, Mr Henry Sleeper, sex=male), PersonImpl(age=49.0, home=New York, NY, name=Harper, Mrs Henry Sleeper (Myna Haxtun), sex=female), PersonImpl(age=45.0, home=New York, NY, name=Harris, Mr Henry Birkhardt, sex=male), PersonImpl(age=36.0, home=New York, NY, name=Harris, Mrs Henry Birkhardt (Irene Wallach), sex=female), PersonImpl(age=55.0, home=Montreal, PQ, name=Hays, Mr Charles Melville, sex=male), PersonImpl(age=52.0, home=Montreal, PQ, name=Hays, Mrs Charles Melville (Clara Jennings Gregg), sex=female), PersonImpl(age=24.0, home=New York, NY, name=Hays, Miss Margaret Bechstein, sex=female), PersonImpl(age=16.0, home=Chicago, IL, name=Hippach, Miss Jean Gertrude, sex=female), PersonImpl(age=44.0, home=Chicago, IL, name=Hippach, Mrs Louis Albert (Ida Sophia Fischer), sex=female), PersonImpl(age=51.0, home=Hudson, NY, name=Hogeboom, Mrs John C. (Anna Andrews), sex=female), PersonImpl(age=42.0, home=New York, NY, name=Holverson, Mr Alexander Oskar, sex=male), PersonImpl(age=35.0, home=New York, NY, name=Holverson, Mrs Alexander Oskar (Mary Aline Towner), sex=female), PersonImpl(age=35.0, home=Indianapolis, IN, name=Homer, Mr Harry, sex=male), PersonImpl(age=38.0, home=New York, NY / Stamford CT, name=Hoyt, Mr Frederick Maxfield, sex=male), PersonImpl(age=35.0, home=New York, NY / Stamford CT, name=Hoyt, Mrs Frederick Maxfield (Jane Anne Forby), sex=female), PersonImpl(age=50.0, home=Paris, France New York, NY, name=Isham, Miss Anne Elizabeth, sex=female), PersonImpl(age=49.0, home=Liverpool, name=Ismay, Mr Joseph Bruce, sex=male), PersonImpl(age=46.0, home=Bennington, VT, name=Jones, Mr Charles Cresson, sex=male), PersonImpl(age=58.0, home=Buffalo, NY, name=Kent, Mr Edward Austin, sex=male), PersonImpl(age=41.0, home=Southington / Noank, CT, name=Kenyon, Mr Frederick R., sex=male), PersonImpl(age=42.0, home=Boston, MA, name=Kimball, Mr Edwin Nelson Jr., sex=male), PersonImpl(age=40.0, home=Boston, MA, name=Kimball, Mrs Edwin Nelson Jr. (Gertrude Parsons), sex=female), PersonImpl(age=42.0, home=Stockholm, Sweden, name=Lindeberg-Lind, Mr Erik Gustaf, sex=male), PersonImpl(age=55.0, home=Stockholm, Sweden, name=Lindstrom, Mrs Carl Johan (Sigrid Posse), sex=female), PersonImpl(age=50.0, home=Paris, France, name=Lines, Mrs Ernest H. (Elizabeth Lindsey James), sex=female), PersonImpl(age=16.0, home=Paris, France, name=Lines, Miss Mary Conover, sex=female), PersonImpl(age=29.0, home=Springfield, MA, name=Long, Mr Milton Clyde, sex=male), PersonImpl(age=21.0, home=Hudson, NY, name=Longley, Miss Gretchen Fiske, sex=female), PersonImpl(age=30.0, home=London / New York, NY, name=Loring, Mr Joseph Holland, sex=male), PersonImpl(age=15.0, home=St Louis, MO, name=Madill, Miss Georgette Alexandra, sex=female), PersonImpl(age=30.0, home=Brockton, MA, name=Maguire, Mr John Edward, sex=male), PersonImpl(age=46.0, home=Vancouver, BC, name=McCaffry, Mr Thomas Francis, sex=male), PersonImpl(age=54.0, home=Dorchester, MA, name=McCarthy, Mr Timothy J., sex=male), PersonImpl(age=36.0, home=Philadelphia, PA, name=McGough, Mr James R., sex=male), PersonImpl(age=28.0, home=New York, NY, name=Meyer, Mr Edgar Joseph, sex=male), PersonImpl(age=65.0, home=East Bridgewater, MA, name=Millet, Mr Francis Davis, sex=male), PersonImpl(age=33.0, home=Green Bay, WI, name=Minahan, Miss Daisy E., sex=female), PersonImpl(age=44.0, home=Fond du Lac, WI, name=Minahan, Dr William Edward, sex=male), PersonImpl(age=37.0, home=Fond du Lac, WI, name=Minahan, Mrs William Edward (Lillian E. Thorpe), sex=female), PersonImpl(age=55.0, home=Montreal, PQ, name=Molson, Mr Harry Markland, sex=male), PersonImpl(age=47.0, home=Washington, DC, name=Moore, Mr Clarence Bloomfield, sex=male), PersonImpl(age=36.0, home=Brooklyn, NY, name=Natsch, Mr Charles H., sex=male), PersonImpl(age=58.0, home=Lexington, MA, name=Newell, Mr Arthur Webster, sex=male), PersonImpl(age=31.0, home=Lexington, MA, name=Newell, Miss Madeleine, sex=female), PersonImpl(age=23.0, home=Lexington, MA, name=Newell, Miss Marjorie, sex=female), PersonImpl(age=19.0, home=New York, NY, name=Newsom, Miss Helen Monypeny, sex=female), PersonImpl(age=64.0, home=Isle of Wight, England, name=Nicholson, Mr Arthur Ernest, sex=male), PersonImpl(age=64.0, home=Providence, RI, name=Ostby, Mr Engelhart Cornelius, sex=male), PersonImpl(age=22.0, home=Providence, RI, name=Ostby, Miss Helen Raghnild, sex=female), PersonImpl(age=28.0, home=?Havana, Cuba, name=Ovies y Rodriguez, Mr Servando, sex=male), PersonImpl(age=22.0, home=Montreal, PQ, name=Payne, Mr Vivian Ponsonby, sex=male), PersonImpl(age=18.0, home=Madrid, Spain, name=Penasco, Mr Victor de Satode, sex=male), PersonImpl(age=17.0, home=Madrid, Spain, name=Penasco, Mrs Victor de Satode (Josefa de Soto), sex=female), PersonImpl(age=52.0, home=Toronto, ON, name=Peuchen, Major Arthur Godfrey, sex=male), PersonImpl(age=46.0, home=Worcester, MA, name=Porter, Mr Walter Chamberlain, sex=male), PersonImpl(age=56.0, home=Mt Airy, Philadelphia, PA, name=Potter, Mrs Thomas, Jr. (Lily Alexenia Wilson), sex=female), PersonImpl(age=43.0, home=St Louis, MO, name=Robert, Mrs Edward Scott (Elisabeth Walton McMillan), sex=female), PersonImpl(age=31.0, home=Trenton, NJ, name=Roebling, Mr Washington Augustus 2nd, sex=male), PersonImpl(age=33.0, home=Paris, France, name=Rosenbaum (Russell), Miss Edith Louise, sex=female), PersonImpl(age=27.0, home=London Vancouver, BC, name=Rothes, the Countess of (Noel Lucy Martha Dyer-Edwardes), sex=female), PersonImpl(age=55.0, home=New York, NY, name=Rothschild, Mr Martin, sex=male), PersonImpl(age=54.0, home=New York, NY, name=Rothschild, Mrs Martin (Elizabeth L. Barrett), sex=female), PersonImpl(age=61.0, home=Haverford, PA / Cooperstown, NY, name=Ryerson, Mr Arthur Larned, sex=male), PersonImpl(age=48.0, home=Haverford, PA / Cooperstown, NY, name=Ryerson, Mrs Arthur Larned (Emily Maria Borie), sex=female), PersonImpl(age=18.0, home=Haverford, PA / Cooperstown, NY, name=Ryerson, Miss Emily Borie, sex=female), PersonImpl(age=13.0, home=Haverford, PA / Cooperstown, NY, name=Ryerson, Master John Borie, sex=male), PersonImpl(age=21.0, home=Haverford, PA / Cooperstown, NY, name=Ryerson, Miss Susan (Suzette) Parker, sex=female), PersonImpl(age=34.0, home=New York, NY, name=Seward, Mr Frederic Kimber, sex=male), PersonImpl(age=40.0, home=New York, NY / Greenwich CT, name=Shutes, Miss Elizabeth W., sex=female), PersonImpl(age=36.0, home=St Louis, MO, name=Silverthorne, Mr Spencer Victor, sex=male), PersonImpl(age=50.0, home=Duluth, MN, name=Silvey, Mr William Baird, sex=male), PersonImpl(age=39.0, home=Duluth, MN, name=Silvey, Mrs William Baird (Alice Munger), sex=female), PersonImpl(age=56.0, home=Basel, Switzerland, name=Simonius-Blumer, Col Alfons, sex=male), PersonImpl(age=28.0, home=New Britain, CT, name=Sloper, Mr William Thompson, sex=male), PersonImpl(age=56.0, home=New York, NY, name=Smart, Mr John Montgomery, sex=male), PersonImpl(age=56.0, home=St James, Long Island, NY, name=Smith, Mr James Clinch, sex=male), PersonImpl(age=24.0, home=Huntington, WV, name=Smith, Mr Lucien Philip, sex=male), PersonImpl(age=18.0, home=Huntington, WV, name=Smith, Mrs Lucien Philip (Mary Eloise Hughes, sex=female), PersonImpl(age=24.0, home=Minneapolis, MN, name=Snyder, Mr John Pillsbury, sex=male), PersonImpl(age=23.0, home=Minneapolis, MN, name=Snyder, Mrs John Pillsbury (Nelle Stevenson), sex=female), PersonImpl(age=45.0, home=Tuxedo Park, NY, name=Spedden, Mr Frederick Oakley, sex=male), PersonImpl(age=40.0, home=Tuxedo Park, NY, name=Spedden, Mrs Frederick Oakley (Margaretta Corning Stone), sex=female), PersonImpl(age=6.0, home=Tuxedo Park, NY, name=Spedden, Master Robert Douglas, sex=male), PersonImpl(age=57.0, home=Paris, France, name=Spencer, Mr William Augustus, sex=male), PersonImpl(age=32.0, home=Basel, Switzerland, name=Staehlin, Dr Max, sex=male), PersonImpl(age=62.0, home=Wimbledon Park, London / Hayling Island, Hants, name=Stead, Mr William Thomas, sex=male), PersonImpl(age=54.0, home=Newark, NJ, name=Stengel, Mr Charles Emil Henry, sex=male), PersonImpl(age=43.0, home=Newark, NJ, name=Stengel, Mrs Charles Emil Henry (Annie May Morris), sex=female), PersonImpl(age=52.0, home=Haverford, PA, name=Stephenson, Mrs Walter Bertram (Martha Eustis), sex=female), PersonImpl(age=62.0, home=Cincinatti, OH, name=Stone, Mrs George Nelson (Martha E.), sex=female), PersonImpl(age=67.0, home=New York, NY, name=Straus, Mr Isidor, sex=male), PersonImpl(age=63.0, home=New York, NY, name=Straus, Mrs Isidor (Ida Blun), sex=female), PersonImpl(age=61.0, home=Haddenfield, NJ, name=Sutton, Mr Frederick, sex=male), PersonImpl(age=46.0, home=Brooklyn, NY, name=Swift, Mrs Frederick Joel (Margaret Welles Barron), sex=female), PersonImpl(age=52.0, home=New York, NY, name=Taussig, Mr Emil, sex=male), PersonImpl(age=39.0, home=New York, NY, name=Taussig, Mrs Emil (Tillie Mandelbaum), sex=female), PersonImpl(age=18.0, home=New York, NY, name=Taussig, Miss Ruth, sex=female), PersonImpl(age=48.0, home=London / East Orange, NJ, name=Taylor, Mr Elmer Zebley, sex=male), PersonImpl(age=49.0, home=Haverford, PA, name=Thayer, Mr John Borland, sex=male), PersonImpl(age=39.0, home=Haverford, PA, name=Thayer, Mrs John Borland (Marian Longstreth Morris), sex=female), PersonImpl(age=17.0, home=Haverford, PA, name=Thayer, Mr John Borland, jr., sex=male), PersonImpl(age=46.0, home=New York, NY, name=Thorne, Mr George (alias of: Mr George Rosenshine), sex=male), PersonImpl(age=31.0, home=Albany, NY, name=Tucker, Mr Gilbert Milligan, jr, sex=male), PersonImpl(age=61.0, home=Brooklyn, NY, name=Van Derhoef, Mr Wyckoff, sex=male), PersonImpl(age=47.0, home=East Orange, NJ, name=Walker, Mr William Anderson, sex=male), PersonImpl(age=64.0, home=Portland, OR, name=Warren, Mr Frank Manley, sex=male), PersonImpl(age=60.0, home=Portland, OR, name=Warren, Mrs Frank Manley (Anna S. Atkinson), sex=female), PersonImpl(age=60.0, home=England Salt Lake City, Utah, name=Weir, Col John, sex=male), PersonImpl(age=55.0, home=New York, NY / Briarcliff Manor NY, name=White, Mrs J. Stuart (Ella Holmes), sex=female), PersonImpl(age=54.0, home=Brunswick, ME, name=White, Mr Percival Wayland, sex=male), PersonImpl(age=21.0, home=Brunswick, ME, name=White, Mr Richard Frasar, sex=male), PersonImpl(age=57.0, home=Youngstown, OH, name=Wick, Mr George Dennick, sex=male), PersonImpl(age=45.0, home=Youngstown, OH, name=Wick, Mrs George Dennick (Martha Hitchcock), sex=female), PersonImpl(age=31.0, home=Youngstown, OH, name=Wick, Miss Mary Natalie, sex=female), PersonImpl(age=50.0, home=Elkins Park, PA, name=Widener, Mr George Dunton, sex=male), PersonImpl(age=50.0, home=Elkins Park, PA, name=Widener, Mrs George Dunton (Eleanor Elkins), sex=female), PersonImpl(age=27.0, home=Elkins Park, PA, name=Widener, Mr Harry Elkins, sex=male), PersonImpl(age=20.0, home=Duluth, MN, name=Willard, Miss Constance, sex=female), PersonImpl(age=51.0, home=Geneva, Switzerland / Radnor, PA, name=Williams, Mr Charles Duane, sex=male), PersonImpl(age=21.0, home=Geneva, Switzerland / Radnor, PA, name=Williams, Mr Richard Norris II, sex=male), PersonImpl(age=36.0, home=New York, NY / Washington, DC, name=Young, Miss Marie Grice, sex=female), PersonImpl(age=40.0, home=, name=Harrison, Mr William \tHenry, sex=male), PersonImpl(age=32.0, home=, name=Keeping, Mr Edwin, sex=male), PersonImpl(age=33.0, home=, name=Ringhini, Mr Sante, sex=male), PersonImpl(age=30.0, home=Russia New York, NY, name=Abelson, Mr Samuel, sex=male), PersonImpl(age=28.0, home=Russia New York, NY, name=Abelson, Mrs Samuel (Anna), sex=female), PersonImpl(age=18.0, home=Buenos Aires, Argentina / New Jersey, NJ, name=Andrew, Mr Edgar Samuel, sex=male), PersonImpl(age=34.0, home=Warwick, England, name=Angle, Mr William A., sex=male), PersonImpl(age=32.0, home=Warwick, England, name=Angle, Mrs William A. (Florence), sex=female), PersonImpl(age=57.0, home=West Hoboken, NJ, name=Ashby, Mr John, sex=male), PersonImpl(age=18.0, home=Penzance, Cornwall / Akron, OH, name=Bailey, Mr Percy Andrew, sex=male), PersonImpl(age=23.0, home=Guernsey, name=Baimbrigge, Mr Charles R., sex=male), PersonImpl(age=36.0, home=Bristol, Avon / Jacksonville, FL, name=Balls, Mrs Ada E. Hall, sex=female), PersonImpl(age=28.0, home=Plymouth, Dorset / Houghton, MI, name=Banfield, Mr Frederick J., sex=male), PersonImpl(age=51.0, home=Jacksonville, FL, name=Bateman, Rev Robert James, sex=male), PersonImpl(age=32.0, home=Norwich / New York, NY, name=Beane, Mr Edward, sex=male), PersonImpl(age=19.0, home=Norwich / New York, NY, name=Beane, Mrs Edward (Ethel Clarke), sex=female), PersonImpl(age=28.0, home=England, name=Beauchamp, Mr Henry James, sex=male), PersonImpl(age=36.0, home=Guntur, India / Benton Harbour, MI, name=Becker, Mrs Allen Oliver (Nellie E. Baumgardner), sex=female), PersonImpl(age=4.0, home=Guntur, India / Benton Harbour, MI, name=Becker, Miss Marion Louise, sex=female), PersonImpl(age=1.0, home=Guntur, India / Benton Harbour, MI, name=Becker, Master Richard F., sex=male), PersonImpl(age=12.0, home=Guntur, India / Benton Harbour, MI, name=Becker, Miss Ruth Elizabeth, sex=female), PersonImpl(age=34.0, home=London, name=Beesley, Mr Lawrence, sex=male), PersonImpl(age=19.0, home=Rochester, NY, name=Bentham, Miss Lilian W., sex=female), PersonImpl(age=23.0, home=St Ives, Cornwall / Calumet, MI, name=Berriman, Mr William S., sex=male), PersonImpl(age=26.0, home=Elmira, NY / Orange, NJ, name=Botsford, Mr William Hull, sex=male), PersonImpl(age=27.0, home=Lake Arthur, Chavez County, NM, name=Bracken, Mr James H., sex=male), PersonImpl(age=15.0, home=Cape Town, South Africa / Seattle, WA, name=Brown, Miss Edith E., sex=female), PersonImpl(age=45.0, home=Cape Town, South Africa / Seattle, WA, name=Brown, Mr Thomas William Solomon, sex=male), PersonImpl(age=40.0, home=Cape Town, South Africa / Seattle, WA, name=Brown, Mrs Thomas William Solomon (Elizabeth C.), sex=female), PersonImpl(age=20.0, home=Skara, Sweden / Rockford, IL, name=Bryhl, Miss Dagmar, sex=female), PersonImpl(age=25.0, home=Skara, Sweden / Rockford, IL, name=Bryhl, Mr Kurt Arnold Gottfrid, sex=male), PersonImpl(age=36.0, home=Sittingbourne, England / San Diego, CA, name=Buss, Miss Kate, sex=female), PersonImpl(age=25.0, home=Southsea, Hants, name=Butler, Mr Reginald Fenton, sex=male), PersonImpl(age=42.0, home=New York, NY, name=Bystrom, Mrs Carolina, sex=female), PersonImpl(age=26.0, home=Bangkok, Thailand / Roseville, IL, name=Caldwell, Mr Albert Francis, sex=male), PersonImpl(age=26.0, home=Bangkok, Thailand / Roseville, IL, name=Caldwell, Mrs Albert Francis (Sylvia Mae Harbaugh), sex=female), PersonImpl(age=0.8333, home=Bangkok, Thailand / Roseville, IL, name=Caldwell, Master Alden Gates, sex=male), PersonImpl(age=31.0, home=Mamaroneck, NY, name=Cameron, Miss Clear, sex=female), PersonImpl(age=19.0, home=St Ives, Cornwall / Calumet, MI, name=Carbines, Mr William, sex=male), PersonImpl(age=54.0, home=London, name=Carter, Rev Ernest Courtenay, sex=male), PersonImpl(age=44.0, home=London, name=Carter, Mrs Ernest Courtenay (Lillian Hughes), sex=female), PersonImpl(age=52.0, home=Bronx, NY, name=Chapman, Mr Charles Henry, sex=male), PersonImpl(age=30.0, home=Cornwall / Spokane, WA, name=Chapman, Mr John Henry, sex=male), PersonImpl(age=30.0, home=Cornwall / Spokane, WA, name=Chapman, Mrs John Henry (Elizabeth Lawry), sex=female), PersonImpl(age=29.0, home=England / San Francisco, CA, name=Clarke, Mr Charles V., sex=male), PersonImpl(age=29.0, home=Hartford, Huntingdonshire, name=Coleridge, Mr Reginald Charles, sex=male), PersonImpl(age=27.0, home=Helsinki, Finland Ashtabula, Ohio, name=Collander, Mr Erik, sex=male), PersonImpl(age=24.0, home=London / Fort Byron, NY, name=Collett, Mr Sidney C. Stuart, sex=male), PersonImpl(age=35.0, home=Bishopstoke, Hants / Fayette Valley, ID, name=Collyer, Mr Harvey, sex=male), PersonImpl(age=31.0, home=Bishopstoke, Hants / Fayette Valley, ID, name=Collyer, Mrs Harvey (Charlotte Tate), sex=female), PersonImpl(age=8.0, home=Bishopstoke, Hants / Fayette Valley, ID, name=Collyer, Miss Marjorie, sex=female), PersonImpl(age=22.0, home=Pennsylvania, name=Cook, Mrs Selena Rogers, sex=female), PersonImpl(age=30.0, home=Provo, UT, name=Corbett, Mrs Walter H. (Irene Colvin), sex=female), PersonImpl(age=20.0, home=Penzance, Cornwall / Akron, OH, name=Cotterill, Mr Harry, sex=male), PersonImpl(age=21.0, home=Lyndhurst, England, name=Davies, Mr Charles Henry, sex=male), PersonImpl(age=49.0, home=St Ives, Cornwall / Hancock, MI, name=Davis, Mrs Agnes, sex=female), PersonImpl(age=8.0, home=St Ives, Cornwall / Hancock, MI, name=Davis, Master John Morgan, sex=male), PersonImpl(age=28.0, home=London / Staten Island, NY, name=Davis, Miss Mary, sex=female), PersonImpl(age=18.0, home=, name=Deacon, Mr Percy, sex=male), PersonImpl(age=28.0, home=Lucca, Italy / California, name=del Carlo, Mr Sebastiano, sex=male), PersonImpl(age=22.0, home=Lucca, Italy / California, name=del Carlo, Mrs Sebastiano (Argenia Genovese), sex=female), PersonImpl(age=25.0, home=Guernsey / Elizabeth, NJ, name=Denbury, Mr Herbert, sex=male), PersonImpl(age=18.0, home=New Forest, England, name=Dibden, Mr William, sex=male), PersonImpl(age=32.0, home=Southampton, name=Doling, Mrs Ada, sex=female), PersonImpl(age=18.0, home=Southampton, name=Doling, Miss Elsie, sex=female), PersonImpl(age=42.0, home=Greenport, NY, name=Drew, Mr James Vivian, sex=male), PersonImpl(age=34.0, home=Greenport, NY, name=Drew, Mrs James Vivian (Lulu Thorne Christian), sex=female), PersonImpl(age=8.0, home=Greenport, NY, name=Drew, Master Marshall Brines, sex=male), PersonImpl(age=23.0, home=England / Detroit, MI, name=Eitemiller, Mr George Floyd, sex=male), PersonImpl(age=21.0, home=Goteborg, Sweden / Rockford, IL, name=Enander, Mr Ingvar, sex=male), PersonImpl(age=19.0, home=Oslo, Norway Bayonne, NJ, name=Fahlstrom, Mr Arne Jonas, sex=male), PersonImpl(age=38.0, home=Rochester, NY, name=Fox, Mr Stanley H., sex=male), PersonImpl(age=38.0, home=Janjgir, India / Pennsylvania, name=Funk, Miss Annie C., sex=female), PersonImpl(age=35.0, home=Liverpool / Montreal, PQ, name=Fynney, Mr Joseph J., sex=male), PersonImpl(age=35.0, home=Cornwall / Clear Creek, CO, name=Gale, Mr Harry, sex=male), PersonImpl(age=38.0, home=Cornwall / Clear Creek, CO, name=Gale, Mr Shadrach, sex=male), PersonImpl(age=24.0, home=Brooklyn, NY, name=Garside, Miss Ethel, sex=female), PersonImpl(age=16.0, home=Liverpool / Montreal, PQ, name=Gaskell, Mr Alfred, sex=male), PersonImpl(age=26.0, home=Guernsey / Elizabeth, NJ, name=Gavey, Mr Lawrence, sex=male), PersonImpl(age=45.0, home=Cornwall, name=Gilbert, Mr William, sex=male), PersonImpl(age=24.0, home=Cornwall / Camden, NJ, name=Giles, Mr Edgar, sex=male), PersonImpl(age=21.0, home=Cornwall / Camden, NJ, name=Giles, Mr Frederick, sex=male), PersonImpl(age=22.0, home=West Kensington, London, name=Giles, Mr Ralph, sex=male), PersonImpl(age=34.0, home=Vancouver, BC, name=Gillespie, Mr William, sex=male), PersonImpl(age=30.0, home=, name=Givard, Mr Hans Christensen, sex=male), PersonImpl(age=50.0, home=Bronx, NY, name=Greenberg, Mr Samuel, sex=male), PersonImpl(age=30.0, home=Auburn, NY, name=Hale, Mr Reginald, sex=male), PersonImpl(age=23.0, home=Detroit, MI, name=Hamalainen, Mrs William (Anna), sex=female), PersonImpl(age=1.0, home=Detroit, MI, name=Hamalainen, Master Viljo, sex=male), PersonImpl(age=44.0, home=Seattle, WA / Toledo, OH, name=Harbeck, Mr William H., sex=male), PersonImpl(age=28.0, home=Denmark Hill, Surrey / Chicago, name=Harper, Rev John, sex=male), PersonImpl(age=6.0, home=Denmark Hill, Surrey / Chicago, name=Harper, Miss Nina, sex=female), PersonImpl(age=30.0, home=London, name=Harris, Mr George, sex=male), PersonImpl(age=43.0, home=Ilford, Essex / Winnipeg, MB, name=Hart, Mr Benjamin, sex=male), PersonImpl(age=45.0, home=Ilford, Essex / Winnipeg, MB, name=Hart, Mrs Benjamin (Esther), sex=female), PersonImpl(age=7.0, home=Ilford, Essex / Winnipeg, MB, name=Hart, Miss Eva Miriam, sex=female), PersonImpl(age=24.0, home=Somerset / Bernardsville, NJ, name=Herman, Miss Alice, sex=female), PersonImpl(age=24.0, home=Somerset / Bernardsville, NJ, name=Herman, Miss Kate, sex=female), PersonImpl(age=49.0, home=Somerset / Bernardsville, NJ, name=Herman, Mr Samuel, sex=male), PersonImpl(age=48.0, home=Somerset / Bernardsville, NJ, name=Herman, Mrs Samuel (Jane Laver), sex=female), PersonImpl(age=34.0, home=West Hampstead, London / Neepawa, MB, name=Hickman, Mr Leonard Mark, sex=male), PersonImpl(age=32.0, home=West Hampstead, London / Neepawa, MB, name=Hickman, Mr Lewis, sex=male), PersonImpl(age=21.0, home=West Hampstead, London / Neepawa, MB, name=Hickman, Mr Stanley George, sex=male), PersonImpl(age=18.0, home=Kontiolahti, Finland / Detroit, MI, name=Hiltunen, Miss Marta, sex=female), PersonImpl(age=53.0, home=Cornwall / Akron, OH, name=Hocking, Mrs Elizabeth, sex=female), PersonImpl(age=23.0, home=Cornwall / Akron, OH, name=Hocking, Mr George, sex=male), PersonImpl(age=21.0, home=Cornwall / Akron, OH, name=Hocking, Miss Ellen (Nellie), sex=female), PersonImpl(age=52.0, home=Southampton, name=Hodges, Mr Henry Price, sex=male), PersonImpl(age=42.0, home=England / Sacramento, CA, name=Hold, Mr Stephen, sex=male), PersonImpl(age=36.0, home=England / Sacramento, CA, name=Hold, Mrs Stephen (Annie Margaret), sex=female), PersonImpl(age=21.0, home=New Forest, England, name=Hood, Mr Ambrose, Jr, sex=male), PersonImpl(age=41.0, home=Tokyo, Japan, name=Hosono, Mr Masafumi, sex=male), PersonImpl(age=33.0, home=Philadelphia, PA, name=Hunt, Mr George Henry, sex=male), PersonImpl(age=17.0, home=Guernsey, name=Ilett, Miss Bertha, sex=female), PersonImpl(age=23.0, home=New York, NY, name=Jerwan, Mrs Amin S. (Marie Thuillard), sex=female), PersonImpl(age=34.0, home=Moscow / Bronx, NY, name=Kantor, Mr Sinai, sex=male), PersonImpl(age=22.0, home=India / Pittsburgh, PA, name=Karnes, Mrs J. Frank (Claire Bennett), sex=female), PersonImpl(age=45.0, home=London / New York, NY, name=Kelly, Mrs Florence (Fannie), sex=female), PersonImpl(age=31.0, home=Sweden / Arlington, NJ, name=Kvillner, Mr Johan Henrik Johannesson, sex=male), PersonImpl(age=30.0, home=Minneapolis, MN, name=Lahtinen, Rev William, sex=male), PersonImpl(age=26.0, home=Minneapolis, MN, name=Lahtinen, Mrs William (Anna Sylvan), sex=female), PersonImpl(age=34.0, home=Chicago, IL, name=Lemore, Mrs Amelia, sex=female), PersonImpl(age=26.0, home=Paris / Haiti, name=LaRoche, Mr Joseph, sex=male), PersonImpl(age=22.0, home=Paris / Haiti, name=LaRoche, Mrs Joseph (Juliet), sex=female), PersonImpl(age=1.0, home=Paris / Haiti, name=LaRoche, Miss Louise, sex=female), PersonImpl(age=3.0, home=Paris / Haiti, name=LaRoche, Miss Simonne, sex=female), PersonImpl(age=25.0, home=, name=Leyson, Mr Robert William Norman, sex=male), PersonImpl(age=48.0, home=Weston-Super-Mare, Somerset, name=Louch, Mr Charles Alexander, sex=male), PersonImpl(age=57.0, home=Southampton / New York, NY, name=Mack, Mrs Mary, sex=female), PersonImpl(age=2.0, home=Paris / Montreal, PQ, name=Mallet, Master Andre, sex=male), PersonImpl(age=27.0, home=Worcester, MA, name=Mantvila, Rev Joseph, sex=male), PersonImpl(age=19.0, home=Worcester, England, name=Marshall, Mrs Kate Louise Phillips, sex=female), PersonImpl(age=30.0, home=St Austall, Cornwall, name=Matthews, Mr William John, sex=male), PersonImpl(age=20.0, home=Weston-Super-Mare / Moose Jaw, SK, name=Maybery, Mr Frank H., sex=male), PersonImpl(age=45.0, home=Sydney, Australia, name=McCrae, Mr Arthur Gordon, sex=male), PersonImpl(age=46.0, home=Rochester, NY, name=McKane, Mr Peter D., sex=male), PersonImpl(age=41.0, home=England / Bennington, VT, name=Mellenger, Mrs Elizabeth Anne, sex=female), PersonImpl(age=13.0, home=England / Bennington, VT, name=Mellenger, Miss Madeleine Violet, sex=female), PersonImpl(age=19.0, home=Chelsea, London, name=Mellor, Mr William John, sex=male), PersonImpl(age=30.0, home=Harrow-on-the-Hill, Middlesex, name=Meyer, Mr August, sex=male), PersonImpl(age=48.0, home=Copenhagen, Denmark, name=Milling, Mr Jacob Christian, sex=male), PersonImpl(age=71.0, home=Guernsey / Montclair, NJ and/or Toledo, Ohio, name=Mitchell, Mr Henry Michael, sex=male), PersonImpl(age=54.0, home=Frankfort, KY, name=Moraweck, Dr Ernest, sex=male), PersonImpl(age=64.0, home=Cambridge, MA, name=Myles, Mr Thomas Francis, sex=male), PersonImpl(age=32.0, home=New York, NY, name=Nasser (Nasrallah), Mr Nicholas, sex=male), PersonImpl(age=18.0, home=New York, NY, name=Nasser (Nasrallah), Mrs Nicholas, sex=female), PersonImpl(age=2.0, home=Nice, France, name=Navratil, Master Edmond Roger, sex=male), PersonImpl(age=32.0, home=Nice, France, name=Navratil, Mr Michel, sex=male), PersonImpl(age=3.0, home=Nice, France, name=Navratil, Master Michel M., sex=male), PersonImpl(age=26.0, home=Boston, MA, name=Nesson, Mr Israel, sex=male), PersonImpl(age=19.0, home=Cornwall / Hancock, MI, name=Nicholls, Mr Joseph Charles, sex=male), PersonImpl(age=20.0, home=Cologne, Germany, name=Nourney, Mr Alfred (aka Baron von Drachstedt), sex=male), PersonImpl(age=29.0, home=Folkstone, Kent / New York, NY, name=Nye, Mrs Elizabeth Ramell, sex=female), PersonImpl(age=39.0, home=Middleburg Heights, OH, name=Otter, Mr Richard, sex=male), PersonImpl(age=22.0, home=Pondersend, England / New Durham, NJ, name=Oxenham, Mr Percy Thomas, sex=male), PersonImpl(age=24.0, home=Hamilton, ON, name=Pain, Dr Alfred, sex=male), PersonImpl(age=28.0, home=St Andrews, Guernsey, name=Parker, Mr Clifford R., sex=male), PersonImpl(age=50.0, home=Woodford County, KY, name=Parrish, Mrs Lutie Davis, sex=female), PersonImpl(age=20.0, home=Gunnislake, England / Butte, MT, name=Pengelly, Mr Frederick, sex=male), PersonImpl(age=40.0, home=, name=Peruschitz, Rev. Joseph M., sex=male), PersonImpl(age=42.0, home=Ilfracombe, Devon, name=Phillips, Miss Alice, sex=female), PersonImpl(age=21.0, home=Ilfracombe, Devon, name=Phillips, Mr Robert, sex=male), PersonImpl(age=32.0, home=Russia, name=Pinsky, Miss Rosa, sex=female), PersonImpl(age=34.0, home=Denmark / New York, NY, name=Ponesell, Mr Martin, sex=male), PersonImpl(age=33.0, home=Plymouth, Devon / Detroit, MI, name=Quick, Mrs Frederick C. (Jane Richards), sex=female), PersonImpl(age=2.0, home=Plymouth, Devon / Detroit, MI, name=Quick, Miss Phyllis May, sex=female), PersonImpl(age=8.0, home=Plymouth, Devon / Detroit, MI, name=Quick, Miss Winifred Vera, sex=female), PersonImpl(age=36.0, home=Brighton, Sussex, name=Reeves, Mr David, sex=male), PersonImpl(age=34.0, home=Elizabeth, NJ, name=Renouf, Mr Peter Henry, sex=male), PersonImpl(age=30.0, home=Elizabeth, NJ, name=Renouf, Mrs Peter Henry (Lillian Jefferys), sex=female), PersonImpl(age=28.0, home=Spain, name=Reynaldo, Mrs Encarnacion, sex=female), PersonImpl(age=23.0, home=Paris / Montreal, PQ, name=Richard, Mr Emil, sex=male), PersonImpl(age=0.8333, home=Cornwall / Akron, OH, name=Richards, Master George Sidney, sex=male), PersonImpl(age=25.0, home=Cornwall / Akron, OH, name=Richards, Mrs Sidney (Emily Hocking), sex=female), PersonImpl(age=3.0, home=Cornwall / Akron, OH, name=Richards, Master William Rowe, sex=male), PersonImpl(age=50.0, home=London, England / Marietta, Ohio and Milwaukee, WI, name=Ridsdale, Miss Lucy, sex=female), PersonImpl(age=21.0, home=Guernsey / Wilmington, DE, name=Rugg, Miss Emily, sex=female), PersonImpl(age=25.0, home=Deer Lodge, MT, name=Shelley, Mrs William (Imanita), sex=female), PersonImpl(age=18.0, home=Finland / Minneapolis, MN, name=Silven, Miss Lyyli, sex=female), PersonImpl(age=20.0, home=Cornwall / Hancock, MI, name=Sincock, Miss Maude, sex=female), PersonImpl(age=30.0, home=Finland / Washington, DC, name=Siukonnen, Miss Anna, sex=female), PersonImpl(age=59.0, home=Sault St Marie, ON, name=Sjostedt, Mr Ernst Adolf, sex=male), PersonImpl(age=30.0, home=Halifax, NS, name=Slayter, Miss Hilda Mary, sex=female), PersonImpl(age=35.0, home=Cornwall, name=Slemen, Mr Richard James, sex=male), PersonImpl(age=22.0, home=Newark, NJ, name=Smith (Schmidt), Mr Augustus, sex=male), PersonImpl(age=25.0, home=Cornwall / Houghton, MI, name=Sobey, Mr Hayden, sex=male), PersonImpl(age=41.0, home=New York, NY, name=Stanton, Mr Samuel Ward, sex=male), PersonImpl(age=25.0, home=Catford, Kent / Detroit, MI, name=Stokes, Mr Philip Joseph, sex=male), PersonImpl(age=14.0, home=Somerset / Bernardsville, NJ, name=Sweet, Mr George, sex=male), PersonImpl(age=50.0, home=Indianapolis, IN, name=Toomey, Miss Ellen, sex=female), PersonImpl(age=22.0, home=, name=Troupiansky, Mr Moses Aaron, sex=male), PersonImpl(age=27.0, home=Bath, England / Massachusetts, name=Troutt, Miss Edwina Celia, sex=female), PersonImpl(age=29.0, home=Plymouth, England, name=Turpin, Mr William John, sex=male), PersonImpl(age=27.0, home=Plymouth, England, name=Turpin, Mrs William John (Dorothy Anne Wonnacott), sex=female), PersonImpl(age=30.0, home=Barre, Co Washington, VT, name=Veale, Mr James, sex=male), PersonImpl(age=22.0, home=Antwerp, Belgium / Stanton, OH, name=Waelens, Mr Achille, sex=male), PersonImpl(age=35.0, home=Mamaroneck, NY, name=Walcroft, Miss Nellie, sex=female), PersonImpl(age=30.0, home=Bristol, England / New Britain, CT, name=Ware, Mr John James, sex=male), PersonImpl(age=28.0, home=Bristol, England / New Britain, CT, name=Ware, Mrs John James (Florence Louise Long), sex=female), PersonImpl(age=23.0, home=, name=Ware, Mr William J., sex=male), PersonImpl(age=12.0, home=Aberdeen / Portland, OR, name=Watt, Miss Bertha, sex=female), PersonImpl(age=40.0, home=Aberdeen / Portland, OR, name=Watt, Mrs James (Bessie Inglis Milne), sex=female), PersonImpl(age=36.0, home=England / Hartford, CT, name=Webber, Miss Susan, sex=female), PersonImpl(age=28.0, home=Bromsgrove, England / Montreal, PQ, name=Weisz, Mr Leopold, sex=male), PersonImpl(age=32.0, home=Bromsgrove, England / Montreal, PQ, name=Weisz, Mrs Leopold (Mathilde), sex=female), PersonImpl(age=29.0, home=Cornwall / Akron, OH, name=Wells, Mrs Arthur H. (Addie Trevaskis), sex=female), PersonImpl(age=4.0, home=Cornwall / Akron, OH, name=Wells, Miss Joan, sex=female), PersonImpl(age=2.0, home=Cornwall / Akron, OH, name=Wells, Master Ralph Lester, sex=male), PersonImpl(age=36.0, home=Bournmouth, England, name=West, Mr Edwy Arthur, sex=male), PersonImpl(age=33.0, home=Bournmouth, England, name=West, Mrs Edwy Arthur (Ada Mary), sex=female), PersonImpl(age=32.0, home=London, England, name=Wilhelms, Mr Charles, sex=male), PersonImpl(age=26.0, home=Yoevil, England / Cottage Grove, OR, name=Wright, Miss Marion, sex=female), PersonImpl(age=30.0, home=Bryn Mawr, PA, USA, name=Aldworth, Mr Charles Augustus, sex=male), PersonImpl(age=24.0, home=London / Montreal, PQ, name=Brown, Miss Mildred, sex=female), PersonImpl(age=18.0, home=, name=Swane, Mr George, sex=male), PersonImpl(age=42.0, home=, name=Abbing, Mr Anthony, sex=male), PersonImpl(age=13.0, home=East Providence, RI, name=Abbott, Master Eugene Joseph, sex=male), PersonImpl(age=16.0, home=East Providence, RI, name=Abbott, Mr Rossmore Edward, sex=male), PersonImpl(age=35.0, home=East Providence, RI, name=Abbott, Mrs Stanton (Rosa), sex=female), PersonImpl(age=16.0, home=Norway Los Angeles, CA, name=Abelseth, Miss Anna Karen, sex=female), PersonImpl(age=25.0, home=Perkins County, SD, name=Abelseth, Mr Olaus, sex=male), PersonImpl(age=18.0, home=Greensburg, PA, name=Abraham, Mrs Joseph (Sophie Easu), sex=female), PersonImpl(age=20.0, home=Taalintehdas, Finland Hoboken, NJ, name=Abrahamsson, Mr August, sex=male), PersonImpl(age=30.0, home=Asarum, Sweden Brooklyn, NY, name=Adahl, Mr Mauritz Nils Martin, sex=male), PersonImpl(age=26.0, home=Bournemouth, England, name=Adams, Mr John, sex=male), PersonImpl(age=40.0, home=Sweden Akeley, MN, name=Ahlin, Mrs Johanna Persdotter, sex=female), PersonImpl(age=24.0, home=, name=Ahmed, Mr Ali, sex=male), PersonImpl(age=41.0, home=Finland Sudbury, ON, name=Aijo-Nirva, Mr Isak, sex=male), PersonImpl(age=18.0, home=London, England Norfolk, VA, name=Aks, Mrs Sam (Leah Rosen), sex=female), PersonImpl(age=0.8333, home=London, England Norfolk, VA, name=Aks, Master Philip, sex=male), PersonImpl(age=23.0, home=England Albion, NY, name=Alexander, Mr William, sex=male), PersonImpl(age=20.0, home=Salo, Finland Astoria, OR, name=Alhomaki, Mr Ilmari Rudolf, sex=male), PersonImpl(age=25.0, home=Argentina, name=Ali, Mr William, sex=male), PersonImpl(age=35.0, home=Lower Clapton, Middlesex or Erdington, Birmingham, name=Allen, Mr William Henry, sex=male), PersonImpl(age=17.0, home=Windsor, England New York, NY, name=Allum, Mr Owen George, sex=male), PersonImpl(age=32.0, home=Bergen, Norway, name=Andersen, Mr Albert Karvin, sex=male), PersonImpl(age=20.0, home=Oslo, Norway Cameron, WI, name=Andersen, Mr Thor Olsvigen, sex=male), PersonImpl(age=39.0, home=Sweden Winnipeg, MN, name=Andersson, Mr Anders Johan, sex=male), PersonImpl(age=39.0, home=Sweden Winnipeg, MN, name=Andersson, Mrs Anders Johan (Alfrida K. Brogren), sex=female), PersonImpl(age=6.0, home=Sweden Winnipeg, MN, name=Andersson, Miss Ebba Iris, sex=female), PersonImpl(age=2.0, home=Sweden Winnipeg, MN, name=Andersson, Miss Ellis Anna Maria, sex=female), PersonImpl(age=17.0, home=Ruotsinphyhtaa, Finland New York, NY, name=Andersson, Miss Erna, sex=female), PersonImpl(age=38.0, home=Vadsbro, Sweden Ministee, MI, name=Andersson, Miss Ida Augusta Margareta, sex=female), PersonImpl(age=9.0, home=Sweden Winnipeg, MN, name=Andersson, Miss Ingeborg Constancia, sex=female), PersonImpl(age=26.0, home=Hartford, CT, name=Andersson, Mr Johan Samuel, sex=male), PersonImpl(age=11.0, home=Sweden Winnipeg, MN, name=Andersson, Miss Sigrid Elizabeth, sex=female), PersonImpl(age=4.0, home=Sweden Winnipeg, MN, name=Andersson, Master Sigvard Harald Elias, sex=male), PersonImpl(age=20.0, home=Sweden Chicago, IL, name=Andreasson, Mr Paul Edvin, sex=male), PersonImpl(age=26.0, home=Bulgaria Chicago, IL, name=Angheloff, Mr Minko, sex=male), PersonImpl(age=25.0, home=Altdorf, Switzerland, name=Arnold, Mr Josef, sex=male), PersonImpl(age=18.0, home=Altdorf, Switzerland, name=Arnold, Mrs Josef (Josephine Frank), sex=female), PersonImpl(age=24.0, home=Sweden Joliet, IL, name=Aronsson, Mr Ernst Axel Algot, sex=male), PersonImpl(age=35.0, home=, name=Asim, Mr Adola, sex=male), PersonImpl(age=40.0, home=Sweden Worcester, MA, name=Asplund, Mr Carl Oscar Vilhelm Gustafsson, sex=male), PersonImpl(age=38.0, home=Sweden Worcester, MA, name=Asplund, Mrs Carl Oscar (Selma Augusta Johansson), sex=female), PersonImpl(age=5.0, home=Sweden Worcester, MA, name=Asplund, Master Carl Edgar, sex=male), PersonImpl(age=9.0, home=Sweden Worcester, MA, name=Asplund, Master Clarence Gustaf Hugo, sex=male), PersonImpl(age=3.0, home=Sweden Worcester, MA, name=Aspland, Master Edvin Rojj Felix, sex=male), PersonImpl(age=13.0, home=Sweden Worcester, MA, name=Asplund, Master Filip Oscar, sex=male), PersonImpl(age=23.0, home=Oskarshamn, Sweden Minneapolis, MN, name=Asplund, Mr John Charles, sex=male), PersonImpl(age=5.0, home=Sweden Worcester, MA, name=Asplund, Miss Lillian Gertrud, sex=female), PersonImpl(age=45.0, home=Ottawa, ON, name=Assaf, Mrs Mariana, sex=female), PersonImpl(age=23.0, home=, name=Assam, Mr Ali, sex=male), PersonImpl(age=17.0, home=, name=Attalah, Miss Malaka, sex=female), PersonImpl(age=27.0, home=Ottawa, ON, name=Attala (Kalil), Mr Solomon, sex=male), PersonImpl(age=23.0, home=Krakoryd, Sweden Bloomington, IL, name=Augustsson, Mr Albert, sex=male), PersonImpl(age=20.0, home=, name=Baccos, Mr Rafoul, sex=male), PersonImpl(age=32.0, home=Ruotsinphytaa, Finland New York, NY, name=Backstrom, Mr Karl Alfred, sex=male), PersonImpl(age=33.0, home=Ruotsinphytaa, Finland New York, NY, name=Backstrom, Mrs Karl Alfred (Maria Mathilda Gustafsson), sex=female), PersonImpl(age=3.0, home=Syria New York, NY, name=Baclini, Miss Eugenie, sex=female), PersonImpl(age=18.0, home=London Skanteales, NY, name=Badman, Miss Emily Louisa, sex=female), PersonImpl(age=40.0, home=, name=Badt, Mr Mohamed, sex=male), PersonImpl(age=26.0, home=, name=Balkic, Mr Cerin, sex=male), PersonImpl(age=15.0, home=Syria Youngstown, OH, name=Banoura, Miss Ayout, sex=female), PersonImpl(age=45.0, home=Syria Ottawa, ON, name=Barbara, Mrs Catherine, sex=female), PersonImpl(age=18.0, home=Syria Ottawa, ON, name=Barbara, Miss Saude, sex=female), PersonImpl(age=27.0, home=New York, NY, name=Barry, Miss Julia, sex=female), PersonImpl(age=22.0, home=England New York, NY, name=Barton Mr David, sex=male), PersonImpl(age=19.0, home=England, name=Beavan, Mr William Thomas, sex=male), PersonImpl(age=26.0, home=Krakudden, Sweden Moune, IL, name=Bengtsson, Mr John Viktor, sex=male), PersonImpl(age=22.0, home=Tranvik, Finland New York, name=Berglund. Mr Karl Ivar Sven, sex=male), PersonImpl(age=20.0, home=Syria, name=Betros, Mr Tannous, sex=male), PersonImpl(age=32.0, home=Hong Kong New York, NY, name=Bing, Mr Lee, sex=male), PersonImpl(age=21.0, home=Brennes, Norway New York, name=Birkeland, Mr Hans, sex=male), PersonImpl(age=18.0, home=Stockholm, Sweden New York, name=Bjorklund, Ernst Herbert, sex=male), PersonImpl(age=26.0, home=Bulgaria Chicago, IL, name=Bostandyeff, Mr Guentcho, sex=male), PersonImpl(age=6.0, home=Syria Kent, ON, name=Boulos, Master Akar, sex=male), PersonImpl(age=9.0, home=Syria Kent, ON, name=Boulos, Miss Laura, sex=female), PersonImpl(age=40.0, home=Ireland Chicago, IL, name=Bourke, Mr John, sex=male), PersonImpl(age=32.0, home=Ireland Chicago, IL, name=Bourke, Mrs John (Catherine), sex=female), PersonImpl(age=26.0, home=Treherbert, Cardiff, Wales, name=Bowen, Mr David, sex=male), PersonImpl(age=18.0, home=Kingwilliamstown, Co Cork, Ireland Glens Falls, NY, name=Bradley, Miss Bridget Delia, sex=female), PersonImpl(age=20.0, home=Medeltorp, Sweden Chicago, IL, name=Braf, Miss Elin Ester Maria, sex=female), PersonImpl(age=29.0, home=Bridgerule, Devon, name=Braund, Mr Lewis Richard, sex=male), PersonImpl(age=22.0, home=Bridgerule, Devon, name=Braund, Mr Owen Harris, sex=male), PersonImpl(age=22.0, home=Sweden Worcester, MA, name=Brobek, Mr Karl Rudolf, sex=male), PersonImpl(age=35.0, home=Broomfield, Chelmsford, England, name=Brocklebank, Mr William Alfred, sex=male), PersonImpl(age=21.0, home=Kingwilliamstown, Co Cork, Ireland New York, NY, name=Buckley, Mr Daniel, sex=male), PersonImpl(age=20.0, home=Co Cork, Ireland Roxbury, MA, name=Buckley, Miss Katherine, sex=female), PersonImpl(age=19.0, home=Co Cork, Ireland Charlestown, MA, name=Burke, Mr Jeremiah, sex=male), PersonImpl(age=18.0, home=Co Sligo, Ireland New York, NY, name=Burns, Miss Mary Delia, sex=female), PersonImpl(age=18.0, home=Croatia, name=Cacic, Mr Grego, sex=male), PersonImpl(age=38.0, home=Croatia, name=Cacic, Mr Luka, sex=male), PersonImpl(age=30.0, home=Croatia, name=Cacic, Mr Maria, sex=male), PersonImpl(age=17.0, home=, name=Calic, Mr Peter, sex=male), PersonImpl(age=21.0, home=, name=Canavan, Miss Mary, sex=female), PersonImpl(age=21.0, home=Ireland Philadelphia, PA, name=Canavan, Mr Patrick, sex=male), PersonImpl(age=21.0, home=, name=Cann, Mr Ernest, sex=male), PersonImpl(age=24.0, home=Goteborg, Sweden Huntley, IL, name=Carlsson, Mr Carl Robert, sex=male), PersonImpl(age=33.0, home=New York, NY, name=Carlsson, Mr Frans Olof, sex=male), PersonImpl(age=33.0, home=, name=Carlsson, Mr Julius, sex=male), PersonImpl(age=28.0, home=Dagsas, Sweden Fower, MN, name=Carlsson, Mr August Sigfrid, sex=male), PersonImpl(age=16.0, home=Co Longford, Ireland New York, NY, name=Carr, Miss Helen, sex=female), PersonImpl(age=37.0, home=Co Sligo, Ireland Hartford, CT, name=Carr, Miss Jeannie, sex=female), PersonImpl(age=28.0, home=St Denys, Southampton, Hants, name=Carver, Mr Alfred John, sex=male), PersonImpl(age=24.0, home=London, name=Celotti, Mr Francesco, sex=male), PersonImpl(age=21.0, home=Ireland New York, NY, name=Chartens, Mr David, sex=male), PersonImpl(age=32.0, home=Hong Kong New York, NY, name=Chip, Mr Chang, sex=male), PersonImpl(age=29.0, home=, name=Christmann, Mr Emil, sex=male), PersonImpl(age=26.0, home=Greece, name=Chronopoulos, Mr Apostolos, sex=male), PersonImpl(age=18.0, home=Greece, name=Chronopoulos, Mr Demetrios, sex=male), PersonImpl(age=20.0, home=Portugal, name=Coelho, Mr Domingos Fernandes, sex=male), PersonImpl(age=19.0, home=London Brooklyn, NY, name=Cohen, Mr Gurshon (Gus), sex=male), PersonImpl(age=24.0, home=Co Limerick, Ireland Sherbrooke, PQ, name=Colbert, Mr Patrick, sex=male), PersonImpl(age=24.0, home=, name=Coleff, Mr Fotio, sex=male), PersonImpl(age=36.0, home=Bulgaria Chicago, IL, name=Coleff, Mr Peyo, sex=male), PersonImpl(age=31.0, home=Philadelphia, PA, name=Conlin, Mr Thomas Henry, sex=male), PersonImpl(age=31.0, home=Ireland Brooklyn, NY, name=Connaghton, Mr Michael, sex=male), PersonImpl(age=30.0, home=Ireland, name=Connolly, Miss Kate, sex=female), PersonImpl(age=22.0, home=Ireland, name=Connolly, Miss Kate, sex=female), PersonImpl(age=43.0, home=, name=Cook, Mr Jacob, sex=male), PersonImpl(age=35.0, home=Austria, name=Cor, Mr Bartol, sex=male), PersonImpl(age=27.0, home=Austria, name=Cor, Mr Ivan, sex=male), PersonImpl(age=19.0, home=Austria, name=Cor, Mr Ludovik, sex=male), PersonImpl(age=30.0, home=London, name=Corn, Mr Harry, sex=male), PersonImpl(age=36.0, home=England Brooklyn, NY, name=Coutts, Mrs William (Minnie), sex=female), PersonImpl(age=3.0, home=England Brooklyn, NY, name=Coutts, Master Neville, sex=male), PersonImpl(age=9.0, home=England Brooklyn, NY, name=Coutts, Master William Leslie, sex=male), PersonImpl(age=59.0, home=Merrill, WI, name=Coxon, Mr Daniel, sex=male), PersonImpl(age=19.0, home=Bristol, England Cleveland, OH, name=Crease, Mr Ernest James, sex=male), PersonImpl(age=44.0, home=Bournemouth, England Newark, NJ, name=Cribb, Mr John Hatfield, sex=male), PersonImpl(age=17.0, home=Bournemouth, England Newark, NJ, name=Cribb, Miss Laura Alice, sex=female), PersonImpl(age=45.0, home=Australia Fingal, ND, name=Dahl, Mr Charles Edward, sex=male), PersonImpl(age=22.0, home=Norrlot, Sweden Chicago, IL, name=Dahlberg, Miss Gerda Ulrika, sex=female), PersonImpl(age=19.0, home=Austria, name=Dakic, Mr Branko, sex=male), PersonImpl(age=29.0, home=Co Athlone, Ireland New York, NY, name=Daly, Mr Eugene, sex=male), PersonImpl(age=30.0, home=Co Athlone, Ireland New York, NY, name=Daly, Miss Marcella, sex=female), PersonImpl(age=34.0, home=Stanton, IA, name=Danbom, Mr Ernst Gilbert, sex=male), PersonImpl(age=28.0, home=Stanton, IA, name=Danbom, Mrs Ernst Gilbert (Anna Sigrid Maria Brogren), sex=female), PersonImpl(age=0.3333, home=Stanton, IA, name=Danbom, Master Gilbert Sigvard Emanuel, sex=male), PersonImpl(age=27.0, home=Bulgaria Chicago, IL, name=Danoff, Mr Yoto, sex=male), PersonImpl(age=25.0, home=Bulgaria Chicago, IL, name=Dantchoff, Mr Khristo, sex=male), PersonImpl(age=24.0, home=West Bromwich, England Pontiac, MI, name=Davies, Mr Alfred, sex=male), PersonImpl(age=22.0, home=, name=Davies, Mr Evan, sex=male), PersonImpl(age=21.0, home=West Bromwich, England Pontiac, MI, name=Davies, Mr John, sex=male), PersonImpl(age=17.0, home=West Bromwich, England Pontiac, MI, name=Davies, Mr Joseph, sex=male), PersonImpl(age=26.0, home=Devon, England Wichita, KS, name=Dean, Mr Bertram, sex=male), PersonImpl(age=33.0, home=Devon, England Wichita, KS, name=Dean, Mrs Bertram (Eva), sex=female), PersonImpl(age=1.0, home=Devon, England Wichita, KS, name=Dean, Master Bertram Vere, sex=male), PersonImpl(age=0.1667, home=Devon, England Wichita, KS, name=Dean, Miss Elizabeth Gladys (Millvena), sex=female), PersonImpl(age=25.0, home=, name=Delalic, Mr Regyo, sex=male), PersonImpl(age=36.0, home=Tampico, MT, name=De Messemaeker, Mr William Joseph, sex=male), PersonImpl(age=36.0, home=Tampico, MT, name=De Messemaeker, Mrs William Joseph (Anna), sex=female), PersonImpl(age=30.0, home=Belgium Detroit, MI, name=De Mulder, Mr Theo, sex=male), PersonImpl(age=23.0, home=, name=Dennis, Mr Samuel, sex=male), PersonImpl(age=26.0, home=, name=Dennis, Mr William, sex=male), PersonImpl(age=19.0, home=Kilmacowen, Co Sligo, Ireland New York, NY, name=Devaney, Miss Margaret, sex=female), PersonImpl(age=65.0, home=, name=Dewan, Mr Frank, sex=male), PersonImpl(age=42.0, home=, name=Dimic, Mr Jovan, sex=male), PersonImpl(age=43.0, home=, name=Dintcheff, Mr Valtcho, sex=male), PersonImpl(age=32.0, home=Ireland New York, NY, name=Dooley, Mr Patrick, sex=male), PersonImpl(age=19.0, home=England Oglesby, IL, name=Dorkings, Mr Edward Arthur, sex=male), PersonImpl(age=30.0, home=Union Hill, NJ, name=Dowdell, Miss Elizabeth, sex=female), PersonImpl(age=24.0, home=Ireland New York, NY, name=Doyle, Miss Elizabeth, sex=female), PersonImpl(age=23.0, home=London New York, NY, name=Drapkin, Miss Jennie, sex=female), PersonImpl(age=24.0, home=Ballydehob, Co Cork, Ireland New York, NY, name=Driscoll, Miss Bridget, sex=female), PersonImpl(age=24.0, home=England Albion, NY, name=Duquemin, Mr Joseph, sex=male), PersonImpl(age=23.0, home=West Haven, CT, name=Dyker, Mr Adolf Fredrik, sex=male), PersonImpl(age=22.0, home=West Haven, CT, name=Dyker, Mrs Adolf Fredrik (Anna Elizabeth Judith Andersson), sex=female), PersonImpl(age=18.0, home=Tofta, Sweden Joliet, IL, name=Edvardsson, Mr Gustaf Hjalmar, sex=male), PersonImpl(age=16.0, home=Karberg, Sweden Jerome Junction, AZ, name=Eklund, Mr Hans Linus, sex=male), PersonImpl(age=45.0, home=Effington Rut, SD, name=Ekstrom, Mr Johan, sex=male), PersonImpl(age=47.0, home=Illinois, USA, name=Elsbury, Mr James, sex=male), PersonImpl(age=5.0, home=New York, NY, name=Emanuel, Miss Virginia Ethel, sex=female), PersonImpl(age=21.0, home=Rotherfield, Sussex, England Essex Co, MA, name=Ford, Miss Doolina Margaret, sex=female), PersonImpl(age=18.0, home=Rotherfield, Sussex, England Essex Co, MA, name=Ford, Mr Edward Watson, sex=male), PersonImpl(age=9.0, home=Rotherfield, Sussex, England Essex Co, MA, name=Ford, Miss Maggie, sex=female), PersonImpl(age=48.0, home=Rotherfield, Sussex, England Essex Co, MA, name=Ford, Mrs Edward (Margaret Ann), sex=female), PersonImpl(age=16.0, home=Rotherfield, Sussex, England Essex Co, MA, name=Ford, Mr Neil Watson, sex=male), PersonImpl(age=25.0, home=New York, NY, name=Gallagher, Mr Martin, sex=male), PersonImpl(age=22.0, home=, name=Gilinski, Mr Leslie, sex=male), PersonImpl(age=16.0, home=Co Longford, Ireland New York, NY, name=Gilnagh, Miss Katie, sex=female), PersonImpl(age=33.0, home=Strood, Kent, England Detroit, MI, name=Goldsmith, Mr Frank John, sex=male), PersonImpl(age=9.0, home=Strood, Kent, England Detroit, MI, name=Goldsmith, Master Frank John William, sex=male), PersonImpl(age=41.0, home=Philadelphia, PA, name=Goldsmith, Mr Nathan, sex=male), PersonImpl(age=38.0, home=Portugal, name=Goncalves, Mr Manuel Estanslas, sex=male), PersonImpl(age=40.0, home=Wiltshire, England Niagara Falls, NY, name=Goodwin, Mr Frederick, sex=male), PersonImpl(age=43.0, home=Wiltshire, England Niagara Falls, NY, name=Goodwin, Mrs Frederick (Augusta), sex=female), PersonImpl(age=14.0, home=Wiltshire, England Niagara Falls, NY, name=Goodwin, Mr Charles E., sex=male), PersonImpl(age=16.0, home=Wiltshire, England Niagara Falls, NY, name=Goodwin, Miss Lillian A., sex=female), PersonImpl(age=9.0, home=Wiltshire, England Niagara Falls, NY, name=Goodwin, Master Harold V., sex=male), PersonImpl(age=10.0, home=Wiltshire, England Niagara Falls, NY, name=Goodwin, Miss Jessie A., sex=female), PersonImpl(age=6.0, home=Wiltshire, England Niagara Falls, NY, name=Goodwin, Master Sidney L., sex=male), PersonImpl(age=11.0, home=Wiltshire, England Niagara Falls, NY, name=Goodwin, Master William F., sex=male), PersonImpl(age=40.0, home=Dorking, Surrey, England, name=Green, Mr George, sex=male), PersonImpl(age=32.0, home=Foresvik, Norway Portland, ND, name=Gronnestad, Mr Daniel Danielsen, sex=male), PersonImpl(age=20.0, home=Waukegan, Chicago, IL, name=Gustafsson, Mr Alfred Ossian, sex=male), PersonImpl(age=37.0, home=Ruotsinphytaa, Finland New York, NY, name=Gustafsson, Mr Anders Vilhelm, sex=male), PersonImpl(age=28.0, home=Ruotsinphytaa, Finland New York, NY, name=Gustafsson, Mr Johan Birger, sex=male), PersonImpl(age=19.0, home=Myren, Sweden New York, NY, name=Gustafsson, Mr Karl Gideon, sex=male)]" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "persons" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Column-based polymorphism" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    namesex
    Allen, Miss Elisabeth Waltonfemale
    Allison, Miss Helen Lorainefemale
    Allison, Mr Hudson Joshua Creightonmale
    Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)female
    Allison, Master Hudson Trevormale
    Anderson, Mr Harrymale
    Andrews, Miss Kornelia Theodosiafemale
    Andrews, Mr Thomas, jrmale
    Appleton, Mrs Edward Dale (Charlotte Lamson)female
    Artagaveytia, Mr Ramonmale
    Astor, Colonel John Jacobmale
    Astor, Mrs John Jacob (Madeleine Talmadge Force)female
    Aubert, Mrs Leontine Paulinefemale
    Barkworth, Mr Algernon H.male
    Baumann, Mr John D.male
    Baxter, Mrs James (Helene DeLaudeniere Chaput)female
    Baxter, Mr Quigg Edmondmale
    Beattie, Mr Thomsonmale
    Beckwith, Mr Richard Leonardmale
    Beckwith, Mrs Richard Leonard (Sallie Monypeny)female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 57, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// When data frame variable is mutable, a strongly typed wrapper for it \n", + "// is generated only once after the first execution of a cell where it is declared\n", + "var nameAndSex = df.select(df.name, df.sex)\n", + "nameAndSex" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    pclassnameembarkedhomeroomticketboatsex
    1stAllen, Miss Elisabeth WaltonSouthamptonSt Louis, MOB-524160 L2212female
    1stAllison, Miss Helen LoraineSouthamptonMontreal, PQ / Chesterville, ONC26female
    1stAllison, Mr Hudson Joshua CreightonSouthamptonMontreal, PQ / Chesterville, ONC26(135)male
    1stAllison, Mrs Hudson J.C. (Bessie Waldo Daniels)SouthamptonMontreal, PQ / Chesterville, ONC26female
    1stAllison, Master Hudson TrevorSouthamptonMontreal, PQ / Chesterville, ONC2211male
    1stAnderson, Mr HarrySouthamptonNew York, NYE-123male
    1stAndrews, Miss Kornelia TheodosiaSouthamptonHudson, NYD-713502 L7710female
    1stAndrews, Mr Thomas, jrSouthamptonBelfast, NIA-36male
    1stAppleton, Mrs Edward Dale (Charlotte Lamson)SouthamptonBayside, Queens, NYC-1012female
    1stArtagaveytia, Mr RamonCherbourgMontevideo, Uruguay(22)male
    1stAstor, Colonel John JacobCherbourgNew York, NY17754 L224 10s 6d(124)male
    1stAstor, Mrs John Jacob (Madeleine Talmadge Force)CherbourgNew York, NY17754 L224 10s 6d4female
    1stAubert, Mrs Leontine PaulineCherbourgParis, FranceB-3517477 L69 6s9female
    1stBarkworth, Mr Algernon H.SouthamptonHessle, YorksA-23Bmale
    1stBaumann, Mr John D.SouthamptonNew York, NYmale
    1stBaxter, Mrs James (Helene DeLaudeniere Chaput)CherbourgMontreal, PQB-58/606female
    1stBaxter, Mr Quigg EdmondCherbourgMontreal, PQB-58/60male
    1stBeattie, Mr ThomsonCherbourgWinnipeg, MNC-6male
    1stBeckwith, Mr Richard LeonardSouthamptonNew York, NYD-355male
    1stBeckwith, Mrs Richard Leonard (Sallie Monypeny)SouthamptonNew York, NYD-355female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// let's declare immutable variable, that contains all string columns\n", + "val strings = df.selectIf{valueClass == String::class}\n", + "strings" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    pclassnameembarkedhomeroomticketboatsex
    1stAllen, Miss Elisabeth WaltonSouthamptonSt Louis, MOB-524160 L2212female
    1stAllison, Miss Helen LoraineSouthamptonMontreal, PQ / Chesterville, ONC26female
    1stAllison, Mr Hudson Joshua CreightonSouthamptonMontreal, PQ / Chesterville, ONC26(135)male
    1stAllison, Mrs Hudson J.C. (Bessie Waldo Daniels)SouthamptonMontreal, PQ / Chesterville, ONC26female
    1stAllison, Master Hudson TrevorSouthamptonMontreal, PQ / Chesterville, ONC2211male
    1stAnderson, Mr HarrySouthamptonNew York, NYE-123male
    1stAndrews, Miss Kornelia TheodosiaSouthamptonHudson, NYD-713502 L7710female
    1stAndrews, Mr Thomas, jrSouthamptonBelfast, NIA-36male
    1stAppleton, Mrs Edward Dale (Charlotte Lamson)SouthamptonBayside, Queens, NYC-1012female
    1stArtagaveytia, Mr RamonCherbourgMontevideo, Uruguay(22)male
    1stAstor, Colonel John JacobCherbourgNew York, NY17754 L224 10s 6d(124)male
    1stAstor, Mrs John Jacob (Madeleine Talmadge Force)CherbourgNew York, NY17754 L224 10s 6d4female
    1stAubert, Mrs Leontine PaulineCherbourgParis, FranceB-3517477 L69 6s9female
    1stBarkworth, Mr Algernon H.SouthamptonHessle, YorksA-23Bmale
    1stBaumann, Mr John D.SouthamptonNew York, NYmale
    1stBaxter, Mrs James (Helene DeLaudeniere Chaput)CherbourgMontreal, PQB-58/606female
    1stBaxter, Mr Quigg EdmondCherbourgMontreal, PQB-58/60male
    1stBeattie, Mr ThomsonCherbourgWinnipeg, MNC-6male
    1stBeckwith, Mr Richard LeonardSouthamptonNew York, NYD-355male
    1stBeckwith, Mrs Richard Leonard (Sallie Monypeny)SouthamptonNew York, NYD-355female

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// 'nameAndSex' is assignable from 'strings', \n", + "// because 'strings' has all the columns that are required by type of 'nameAndSex'\n", + "nameAndSex = strings\n", + "\n", + "// note, that the actual value of 'nameAndSex' is still a data frame of all string columns\n", + "nameAndSex" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "sex [Str][1313]: female, female, male, female, male, male, female, male, female, male, male, female,..." + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// but typed access to the fields works only for 'name' and 'sex'\n", + "nameAndSex.sex // this is OK" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Type mismatch: inferred type is TypedDataFrame but TypedDataFrame was expected" + ] + } + ], + "source": [ + "nameAndSex.home // this fails with compilation error" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "home [Str][1313]: St Louis, MO, Montreal, PQ / Chesterville, ON, Montreal, PQ / Chesterville, ON, Mo..." + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nameAndSex[\"home\"] // the requested column is still available by column name string" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    namehome
    Allen, Miss Elisabeth WaltonSt Louis, MO
    Allison, Miss Helen LoraineMontreal, PQ / Chesterville, ON
    Allison, Mr Hudson Joshua CreightonMontreal, PQ / Chesterville, ON
    Allison, Mrs Hudson J.C. (Bessie Waldo Daniels)Montreal, PQ / Chesterville, ON
    Allison, Master Hudson TrevorMontreal, PQ / Chesterville, ON
    Anderson, Mr HarryNew York, NY
    Andrews, Miss Kornelia TheodosiaHudson, NY
    Andrews, Mr Thomas, jrBelfast, NI
    Appleton, Mrs Edward Dale (Charlotte Lamson)Bayside, Queens, NY
    Artagaveytia, Mr RamonMontevideo, Uruguay
    Astor, Colonel John JacobNew York, NY
    Astor, Mrs John Jacob (Madeleine Talmadge Force)New York, NY
    Aubert, Mrs Leontine PaulineParis, France
    Barkworth, Mr Algernon H.Hessle, Yorks
    Baumann, Mr John D.New York, NY
    Baxter, Mrs James (Helene DeLaudeniere Chaput)Montreal, PQ
    Baxter, Mr Quigg EdmondMontreal, PQ
    Beattie, Mr ThomsonWinnipeg, MN
    Beckwith, Mr Richard LeonardNew York, NY
    Beckwith, Mrs Richard Leonard (Sallie Monypeny)New York, NY

    ... only showing top 20 rows

    " + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "// now let's create a variable with two other columns\n", + "val nameAndHome = df.select(df.name, df.home)\n", + "nameAndHome" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Type mismatch: inferred type is TypedDataFrame but TypedDataFrame was expected" + ] + } + ], + "source": [ + "nameAndSex = nameAndHome // this assignment doesn't work because of columns mismatch" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "// unfortunately, there is a way to get a runtime error here, \n", + "// because typed wrappers are generated only after execution of a cell\n", + "\n", + "// so the following assigment will pass fine, because return type of 'select' is the same as in 'df' variable, \n", + "// although the set of columns was reduced\n", + "nameAndSex = df.select(df.name, df.home) " + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "java.util.NoSuchElementException: Could not find column 'sex' in dataframe\n", + "krangl.SimpleDataFrame.get(SimpleDataFrame.kt:129)\n", + "krangl.typed.TypedDataFrame$DefaultImpls.get(TypedDataFrame.kt:43)\n", + "krangl.typed.TypedDataFrameImpl.get(TypedDataFrame.kt:73)\n", + "Line_184_jupyter.getSex(Unknown Source)\n", + "Line_227_jupyter.(Unknown Source)\n", + "java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n", + "java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n", + "java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n", + "java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:488)\n", + "kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.evalWithConfigAndOtherScriptsResults(BasicJvmScriptEvaluator.kt:95)\n", + "kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.invoke$suspendImpl(BasicJvmScriptEvaluator.kt:40)\n", + "kotlin.script.experimental.jvm.BasicJvmScriptEvaluator.invoke(BasicJvmScriptEvaluator.kt)\n", + "kotlin.script.experimental.jvmhost.repl.JvmReplEvaluator$eval$$inlined$write$lambda$2.invokeSuspend(legacyReplEvaluation.kt:57)\n", + "kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)\n", + "org.jetbrains.kotlin.mainKts.relocatedDeps.kotlinx.coroutines.DispatchedTask.run(Dispatched.kt:238)\n", + "org.jetbrains.kotlin.mainKts.relocatedDeps.kotlinx.coroutines.EventLoopImplBase.processNextEvent(EventLoop.kt:116)\n", + "org.jetbrains.kotlin.mainKts.relocatedDeps.kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt:80)\n", + "org.jetbrains.kotlin.mainKts.relocatedDeps.kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt:54)\n", + "org.jetbrains.kotlin.mainKts.relocatedDeps.kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source)\n", + "org.jetbrains.kotlin.mainKts.relocatedDeps.kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking$default(Builders.kt:36)\n", + "org.jetbrains.kotlin.mainKts.relocatedDeps.kotlinx.coroutines.BuildersKt.runBlocking$default(Unknown Source)\n", + "kotlin.script.experimental.jvmhost.repl.JvmReplEvaluator.eval(legacyReplEvaluation.kt:57)\n", + "org.jetbrains.kotlin.cli.common.repl.ReplEvalAction$DefaultImpls.eval$default(ReplApi.kt:123)\n", + "org.jetbrains.kotlin.jupyter.ReplForJupyterImpl.doEval(repl.kt:429)\n", + "org.jetbrains.kotlin.jupyter.ReplForJupyterImpl.eval(repl.kt:349)\n", + "org.jetbrains.kotlin.jupyter.ProtocolKt$shellMessagesHandler$res$1.invoke(protocol.kt:91)\n", + "org.jetbrains.kotlin.jupyter.ProtocolKt$shellMessagesHandler$res$1.invoke(protocol.kt)\n", + "org.jetbrains.kotlin.jupyter.ProtocolKt.evalWithIO(protocol.kt:252)\n", + "org.jetbrains.kotlin.jupyter.ProtocolKt.shellMessagesHandler(protocol.kt:90)\n", + "org.jetbrains.kotlin.jupyter.IkotlinKt.kernelServer(ikotlin.kt:104)\n", + "org.jetbrains.kotlin.jupyter.IkotlinKt.main(ikotlin.kt:59)\n" + ] + } + ], + "source": [ + "// if we try to access the column, we get runtime error\n", + "nameAndSex.sex " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TODO" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Support operations:\n", + "* Add row\n", + "* Join\n", + "* Reshape\n", + "\n", + "Improve typed wrappers for:\n", + "* Grouped data frame\n", + "* Columns" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Kotlin", + "language": "kotlin", + "name": "kotlin" + }, + "language_info": { + "codemirror_mode": "text/x-kotlin", + "file_extension": ".kt", + "name": "kotlin" + }, + "pycharm": { + "stem_cell": { + "cell_type": "raw", + "source": [], + "metadata": { + "collapsed": false + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle deleted file mode 100644 index f9f67a949..000000000 --- a/settings.gradle +++ /dev/null @@ -1 +0,0 @@ -include 'jupyter-lib' \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 000000000..c8741fc70 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,30 @@ +@file:Suppress("UnstableApiUsage") + +pluginManagement { + val kotlinVersion: String by settings + val shadowJarVersion: String by settings + + repositories { + jcenter() + mavenLocal() + mavenCentral() + maven { url = uri("https://dl.bintray.com/kotlin/kotlin-eap") } + maven { url = uri("https://dl.bintray.com/kotlin/kotlin-dev") } + } + + resolutionStrategy { + eachPlugin { + if (requested.id.id == "com.github.johnrengelman.shadow") { + useModule("com.github.jengelman.gradle.plugins:shadow:$shadowJarVersion") + } + } + } + + plugins { + kotlin("jvm") version kotlinVersion + id("com.github.johnrengelman.shadow") version shadowJarVersion + } + +} + +include("jupyter-lib") diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt new file mode 100644 index 000000000..fdf6f64c5 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/annotationsProcessor.kt @@ -0,0 +1,60 @@ +package org.jetbrains.kotlin.jupyter + +import jupyter.kotlin.KotlinFunctionInfo +import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater + +interface AnnotationsProcessor { + + fun register(handler: TypeHandler): Code + + fun process(line: Any): List +} + +class AnnotationsProcessorImpl(private val contextUpdater: ContextUpdater) : AnnotationsProcessor { + + private val handlers = mutableMapOf() + + private val methodIdMap = mutableMapOf() + + private var nextGeneratedMethodId = 0 + + private fun getMethodName(id: Int) = "___processAnnotation$id" + + override fun register(handler: TypeHandler): Code { + val annotationArgument = "__annotation" + val classArgument = "__class" + val body = handler.code + .replace("\$annotation", annotationArgument) + .replace("\$kclass", classArgument) + val annotationType = handler.className + val methodId = nextGeneratedMethodId++ + val methodName = getMethodName(methodId) + methodIdMap[annotationType] = methodId + return "fun $methodName($annotationArgument : $annotationType, $classArgument : kotlin.reflect.KClass<*>) = $body" + } + + override fun process(line: Any): List { + if (methodIdMap.isNotEmpty()) { + contextUpdater.update() + val resolvedMethods = methodIdMap.map { + it.key to contextUpdater.context.functions[getMethodName(it.value)] + }.filter { it.second != null }.map { it.first to it.second!! } + handlers.putAll(resolvedMethods) + resolvedMethods.forEach { + methodIdMap.remove(it.first) + } + } + val codeToExecute = mutableListOf() + line.javaClass.kotlin.nestedClasses.forEach { kclass -> + kclass.annotations.forEach { + val annotationType = it.annotationClass.qualifiedName!! + val handler = handlers[annotationType] + if (handler != null) { + val result = handler.function.call(handler.line, it, kclass) + (result as? List?)?.let(codeToExecute::addAll) + } + } + } + return codeToExecute + } +} \ No newline at end of file diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt index a84dc77fa..3f892d668 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/commands.kt @@ -11,19 +11,19 @@ fun isCommand(code: String): Boolean = code.startsWith(":") fun Iterable.joinToStringIndented(transform: ((T) -> CharSequence)? = null) = joinToString("\n ", prefix = " ", transform = transform) -fun runCommand(code: String, repl: ReplForJupyter?): ResponseWithMessage { +fun runCommand(code: String, repl: ReplForJupyter?): Response { val args = code.trim().substring(1).split(" ") val cmd = try { ReplCommands.valueOf(args[0]) } catch (e: IllegalArgumentException) { - return ResponseWithMessage(ResponseState.Error, textResult("Failed!"), emptyList(), null, "unknown command: $code\nto see available commands, enter :help") + return AbortResponseWithMessage(textResult("Failed!"), "unknown command: $code\nto see available commands, enter :help") } return when (cmd) { ReplCommands.classpath -> { val cp = repl!!.currentClasspath - ResponseWithMessage(ResponseState.Ok, textResult("Current classpath (${cp.count()} paths):\n${cp.joinToString("\n")}")) + OkResponseWithMessage(textResult("Current classpath (${cp.count()} paths):\n${cp.joinToString("\n")}")) } ReplCommands.help -> { val commands = ReplCommands.values().asIterable().joinToStringIndented { ":${it.name} - ${it.desc}" } @@ -32,10 +32,10 @@ fun runCommand(code: String, repl: ReplForJupyter?): ResponseWithMessage { if (it.argumentsUsage != null) s += "\n Usage: %${it.name} ${it.argumentsUsage}" s } - val libraries = repl?.config?.libraries?.awaitBlocking()?.toList()?.joinToStringIndented { + val libraries = repl?.resolverConfig?.libraries?.awaitBlocking()?.toList()?.joinToStringIndented { "${it.first} ${it.second.link ?: ""}" } - ResponseWithMessage(ResponseState.Ok, textResult("Commands:\n$commands\n\nMagics\n$magics\n\nSupported libraries:\n$libraries")) + OkResponseWithMessage(textResult("Commands:\n$commands\n\nMagics\n$magics\n\nSupported libraries:\n$libraries")) } } } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt index a9be0f786..3b829418f 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/config.kt @@ -23,12 +23,12 @@ val LocalSettingsPath = Paths.get(System.getProperty("user.home"), ".jupyter_kot val GitHubApiHost = "api.github.com" val GitHubRepoOwner = "kotlin" val GitHubRepoName = "kotlin-jupyter" -val GitHubBranchName = "master" +val GitHubBranchName = "psi_completion_with_typing" val GitHubApiPrefix = "https://$GitHubApiHost/repos/$GitHubRepoOwner/$GitHubRepoName" val LibraryDescriptorExt = "json" val LibraryPropertiesFile = ".properties" -val libraryDescriptorFormatVersion = 1 +val libraryDescriptorFormatVersion = 2 internal val log by lazy { LoggerFactory.getLogger("ikotlin") } @@ -47,7 +47,7 @@ data class OutputConfig( var cellOutputMaxSize: Int = 100000, var captureNewlineBufferSize: Int = 100 ) { - fun assign(other: OutputConfig) { + fun update(other: OutputConfig) { captureOutput = other.captureOutput captureBufferTimeLimitMs = other.captureBufferTimeLimitMs captureBufferMaxSize = other.captureBufferMaxSize @@ -77,21 +77,34 @@ data class KernelConfig( val protocolVersion = "5.3" -data class TypeRenderer(val className: String, val displayCode: String?, val resultCode: String?) +data class TypeHandler(val className: TypeName, val code: Code) data class Variable(val name: String, val value: String) -class LibraryDefinition(val dependencies: List, +open class LibraryDefinition( + val dependencies: List, + val initCell: List, + val imports: List, + val repositories: List, + val init: List, + val renderers: List, + val converters: List, + val annotations: List +) + +class LibraryDescriptor(dependencies: List, val variables: List, - val initCell: List, - val imports: List, - val repositories: List, - val init: List, - val renderers: List, - val link: String?) + initCell: List, + imports: List, + repositories: List, + init: List, + renderers: List, + converters: List, + annotations: List, + val link: String?) : LibraryDefinition(dependencies, initCell, imports, repositories, init, renderers, converters, annotations) data class ResolverConfig(val repositories: List, - val libraries: Deferred>) + val libraries: Deferred>) fun parseLibraryArgument(str: String): Variable { val eq = str.indexOf('=') @@ -258,7 +271,9 @@ fun getLibrariesJsons(homeDir: String): Map { } fun loadResolverConfig(homeDir: String) = ResolverConfig(defaultRepositories, GlobalScope.async { - parserLibraryDescriptors(getLibrariesJsons(homeDir)) + log.catchAll { + parserLibraryDescriptors(getLibrariesJsons(homeDir)) + } ?: emptyMap() }) val defaultRepositories = arrayOf( @@ -267,19 +282,22 @@ val defaultRepositories = arrayOf( "https://jitpack.io" ).map { RepositoryCoordinates(it) } -fun parserLibraryDescriptors(libJsons: Map): Map { +fun parserLibraryDescriptors(libJsons: Map): Map { return libJsons.mapValues { - LibraryDefinition( + log.info("Parsing '${it.key}' descriptor") + LibraryDescriptor( dependencies = it.value.array("dependencies")?.toList().orEmpty(), variables = it.value.obj("properties")?.map { Variable(it.key, it.value.toString()) }.orEmpty(), imports = it.value.array("imports")?.toList().orEmpty(), repositories = it.value.array("repositories")?.toList().orEmpty(), init = it.value.array("init")?.toList().orEmpty(), initCell = it.value.array("initCell")?.toList().orEmpty(), - renderers = it.value.array("renderers")?.map { - TypeRenderer(it.string("class")!!, it.string("display"), it.string("result")) + renderers = it.value.obj("renderers")?.map { + TypeHandler(it.key, it.value.toString()) }?.toList().orEmpty(), - link = it.value.string("link") + link = it.value.string("link"), + converters = it.value.obj("typeConverters")?.map { TypeHandler(it.key, it.value.toString()) }.orEmpty(), + annotations = it.value.obj("annotationHandlers")?.map { TypeHandler(it.key, it.value.toString()) }.orEmpty() ) } } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/connection.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/connection.kt index f227b8380..28ddf4d6b 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/connection.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/connection.kt @@ -159,13 +159,15 @@ class HMAC(algo: String, key: String?) { fun ByteArray.toHexString(): String = joinToString("", transform = { "%02x".format(it) }) fun ZMQ.Socket.sendMessage(msg: Message, hmac: HMAC) { - msg.id.forEach { sendMore(it) } - sendMore(DELIM) - val signableMsg = listOf(msg.header, msg.parentHeader, msg.metadata, msg.content) - .map { it?.toJsonString(prettyPrint = false)?.toByteArray() ?: emptyJsonObjectStringBytes } - sendMore(hmac(signableMsg) ?: "") - signableMsg.take(signableMsg.size - 1).forEach { sendMore(it) } - send(signableMsg.last()) + synchronized(this) { + msg.id.forEach { sendMore(it) } + sendMore(DELIM) + val signableMsg = listOf(msg.header, msg.parentHeader, msg.metadata, msg.content) + .map { it?.toJsonString(prettyPrint = false)?.toByteArray() ?: emptyJsonObjectStringBytes } + sendMore(hmac(signableMsg) ?: "") + signableMsg.take(signableMsg.size - 1).forEach { sendMore(it) } + send(signableMsg.last()) + } } fun ZMQ.Socket.receiveMessage(start: ByteArray, hmac: HMAC): Message? { diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt index f342e9d13..6713a64c3 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/ikotlin.kt @@ -80,7 +80,7 @@ fun kernelServer(config: KernelConfig) { val executionCount = AtomicLong(1) - val repl = ReplForJupyter(config.scriptClasspath, config.resolverConfig) + val repl = ReplForJupyterImpl(config.scriptClasspath, config.resolverConfig) val mainThread = Thread.currentThread() @@ -101,7 +101,7 @@ fun kernelServer(config: KernelConfig) { while (true) { try { - conn.shell.onMessage { shellMessagesHandler(it, repl, executionCount) } + conn.shell.onMessage { message -> shellMessagesHandler(message, repl, executionCount) } Thread.sleep(config.pollingIntervalMillis) } catch (e: InterruptedException) { diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt index feb762bdd..2c297d437 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/libraries.kt @@ -1,8 +1,10 @@ package org.jetbrains.kotlin.jupyter -class LibrariesProcessor { +import kotlinx.coroutines.Deferred - data class LibraryWithCode(val library: LibraryDefinition, val code: String) +class LibrariesProcessor(private val libraries: Deferred>?) { + + data class LibraryWithCode(val library: LibraryDescriptor, val code: String) private val processedLibraries = mutableListOf() @@ -41,14 +43,16 @@ class LibrariesProcessor { return result } - private fun generateCode(repl: ReplForJupyter, library: LibraryDefinition, mapping: Map): String { - val builder = StringBuilder() - library.repositories.forEach { builder.appendln("@file:Repository(\"$it\")") } - library.dependencies.forEach { builder.appendln("@file:DependsOn(\"$it\")") } - library.imports.forEach { builder.appendln("import $it") } - library.init.forEach { builder.appendln(repl.preprocessCode(it)) } - return builder.toString().replaceVariables(mapping) - } + private fun processDescriptor(library: LibraryDescriptor, mapping: Map) = LibraryDefinition( + dependencies = library.dependencies.map { replaceVariables(it, mapping) }, + repositories = library.repositories.map { replaceVariables(it, mapping) }, + imports = library.imports.map { replaceVariables(it, mapping) }, + init = library.init.map { replaceVariables(it, mapping) }, + initCell = library.initCell.map { replaceVariables(it, mapping) }, + renderers = library.renderers.map { TypeHandler(it.className, replaceVariables(it.code, mapping)) }, + converters = library.converters.map { TypeHandler(it.className, replaceVariables(it.code, mapping)) }, + annotations = library.annotations.map { TypeHandler(it.className, replaceVariables(it.code, mapping)) } + ) /** * Split a command argument into a set of library calls @@ -83,21 +87,19 @@ class LibrariesProcessor { } } - private fun String.replaceVariables(mapping: Map) = - mapping.asSequence().fold(this) { str, template -> - str.replace("\$${template.key}", template.value) + private fun replaceVariables(str: String, mapping: Map) = + mapping.asSequence().fold(str) { s, template -> + s.replace("\$${template.key}", template.value) } - fun processNewLibraries(repl: ReplForJupyter, arg: String) { - - splitLibraryCalls(arg).forEach { - val (name, vars) = parseLibraryName(it) - val library = repl.config?.libraries?.awaitBlocking()?.get(name) - ?: throw ReplCompilerException("Unknown library '$name'") + fun processNewLibraries(arg: String) = + splitLibraryCalls(arg).map { + val (name, vars) = parseLibraryName(it) + val library = libraries?.awaitBlocking()?.get(name) + ?: throw ReplCompilerException("Unknown library '$name'") - val mapping = substituteArguments(library.variables, vars) + val mapping = substituteArguments(library.variables, vars) - processedLibraries.add(LibraryWithCode(library, generateCode(repl, library, mapping))) - } - } + processDescriptor(library, mapping) + } } \ No newline at end of file diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt index 6de9554ec..876e81370 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/magics.kt @@ -16,89 +16,95 @@ enum class ReplLineMagics(val desc: String, val argumentsUsage: String? = null, output("setup output settings", "--max-cell-size=1000 --no-stdout --max-time=100 --max-buffer=400") } -fun processMagics(repl: ReplForJupyter, code: String): String { +data class MagicProcessingResult(val code: String, val libraries: List = emptyList()) - val sb = StringBuilder() - var nextSearchIndex = 0 - var nextCopyIndex = 0 +class MagicsProcessor(val repl: ReplOptions, val libraries: LibrariesProcessor) { - val outputParser = repl.outputConfig.let { conf -> - object : CliktCommand() { - val defaultConfig = OutputConfig() + fun updateOutputConfig(conf: OutputConfig, argv: List): OutputConfig { + val parser = object : CliktCommand() { val max: Int by option("--max-cell-size", help = "Maximum cell output").int().default(conf.cellOutputMaxSize) val maxBuffer: Int by option("--max-buffer", help = "Maximum buffer size").int().default(conf.captureBufferMaxSize) val maxBufferNewline: Int by option("--max-buffer-newline", help = "Maximum buffer size when newline got").int().default(conf.captureNewlineBufferSize) val maxTimeInterval: Long by option("--max-time", help = "Maximum time wait for output to accumulate").long().default(conf.captureBufferTimeLimitMs) val dontCaptureStdout: Boolean by option("--no-stdout", help = "Don't capture output").flag(default = !conf.captureOutput) val reset: Boolean by option("--reset-to-defaults", help = "Reset to defaults").flag() + override fun run() {} + } + parser.parse(argv) - override fun run() { - if (reset) { - conf.assign(defaultConfig) - return - } - conf.assign( - OutputConfig( - !dontCaptureStdout, - maxTimeInterval, - maxBuffer, - max, - maxBufferNewline - ) + return if (parser.reset) OutputConfig() else + with(parser) { + OutputConfig( + !dontCaptureStdout, + maxTimeInterval, + maxBuffer, + max, + maxBufferNewline ) } - } } - while (true) { - - var magicStart: Int - do { - magicStart = code.indexOf("%", nextSearchIndex) - nextSearchIndex = magicStart + 1 - } while (magicStart != -1 && magicStart != 0 && code[magicStart - 1] != '\n') - if (magicStart == -1) { - sb.append(code.substring(nextCopyIndex)) - return sb.toString() - } + fun processMagics(code: String): MagicProcessingResult { - val magicEnd = code.indexOf('\n', magicStart).let { if (it == -1) code.length else it } - val magicText = code.substring(magicStart + 1, magicEnd) + val sb = StringBuilder() + var nextSearchIndex = 0 + var nextCopyIndex = 0 - try { - val parts = magicText.split(' ', limit = 2) - val keyword = parts[0] - val arg = if (parts.count() > 1) parts[1] else null + val newLibraries = mutableListOf() + while (true) { - val magic = try { - ReplLineMagics.valueOf(keyword) - } catch (e: IllegalArgumentException) { - throw ReplCompilerException("Unknown line magic keyword: '$keyword'") + var magicStart: Int + do { + magicStart = code.indexOf("%", nextSearchIndex) + nextSearchIndex = magicStart + 1 + } while (magicStart != -1 && magicStart != 0 && code[magicStart - 1] != '\n') + if (magicStart == -1) { + sb.append(code.substring(nextCopyIndex)) + return MagicProcessingResult(sb.toString(), newLibraries) } - sb.append(code.substring(nextCopyIndex, magicStart)) + val magicEnd = code.indexOf('\n', magicStart).let { if (it == -1) code.length else it } + val magicText = code.substring(magicStart + 1, magicEnd) - when (magic) { - ReplLineMagics.trackExecution -> repl.trackExecutedCode = true - ReplLineMagics.trackClasspath -> repl.trackClasspath = true - ReplLineMagics.dumpClassesForSpark -> { - val cw = ClassWriter() - System.setProperty("spark.repl.class.outputDir", cw.outputDir.toString()) - repl.classWriter = cw - } - ReplLineMagics.use -> { - if (arg == null) throw ReplCompilerException("Need some arguments for 'use' command") - repl.librariesCodeGenerator.processNewLibraries(repl, arg) + try { + val parts = magicText.split(' ', limit = 2) + val keyword = parts[0] + val arg = if (parts.count() > 1) parts[1] else null + + val magic = try { + ReplLineMagics.valueOf(keyword) + } catch (e: IllegalArgumentException) { + throw ReplCompilerException("Unknown line magic keyword: '$keyword'") } - ReplLineMagics.output -> { - outputParser.parse((arg ?: "").split(" ")) + + sb.append(code.substring(nextCopyIndex, magicStart)) + + when (magic) { + ReplLineMagics.trackExecution -> { + repl.executedCodeLogging = when (arg?.trim()) { + "-all" -> ExecutedCodeLogging.All + "-off" -> ExecutedCodeLogging.Off + "-generated" -> ExecutedCodeLogging.Generated + else -> ExecutedCodeLogging.All + } + } + ReplLineMagics.trackClasspath -> repl.trackClasspath = true + ReplLineMagics.dumpClassesForSpark -> repl.writeCompiledClasses = true + + ReplLineMagics.use -> { + if (arg == null) throw ReplCompilerException("Need some arguments for 'use' command") + newLibraries.addAll(libraries.processNewLibraries(arg)) + } + ReplLineMagics.output -> { + repl.outputConfig = updateOutputConfig(repl.outputConfig, (arg ?: "").split(" ")) + } } + nextCopyIndex = magicEnd + nextSearchIndex = magicEnd + } catch (e: Exception) { + throw ReplCompilerException("Failed to process '%$magicText' command. " + e.message) } - nextCopyIndex = magicEnd - nextSearchIndex = magicEnd - } catch (e: Exception) { - throw ReplCompilerException("Failed to process '%$magicText' command. " + e.message) } } } \ No newline at end of file diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt index cee0ae008..ea6dbb56c 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/protocol.kt @@ -1,9 +1,10 @@ package org.jetbrains.kotlin.jupyter import com.beust.klaxon.JsonObject -import jupyter.kotlin.DisplayResult import jupyter.kotlin.MimeTypedResult import jupyter.kotlin.textResult +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.config.KotlinCompilerVersion import java.io.ByteArrayOutputStream @@ -12,9 +13,10 @@ import java.io.PrintStream import java.lang.reflect.InvocationTargetException import java.util.concurrent.atomic.AtomicLong import kotlin.concurrent.timer +import kotlin.system.exitProcess enum class ResponseState { - Ok, Error + Ok, Error, Abort } enum class JupyterOutType { @@ -22,9 +24,46 @@ enum class JupyterOutType { fun optionName() = name.toLowerCase() } -data class ResponseWithMessage(val state: ResponseState, val result: MimeTypedResult?, val displays: List = emptyList(), val stdOut: String? = null, val stdErr: String? = null) { - val hasStdOut: Boolean = stdOut != null && stdOut.isNotEmpty() - val hasStdErr: Boolean = stdErr != null && stdErr.isNotEmpty() +interface Response { + val state: ResponseState + val hasStdOut: Boolean + val hasStdErr: Boolean + val stdOut: String? + val stdErr: String? +} + +data class OkResponseWithMessage( + val result: MimeTypedResult?, + override val stdOut: String? = null, + override val stdErr: String? = null, +): Response{ + override val state: ResponseState = ResponseState.Ok + override val hasStdOut: Boolean = stdOut != null && stdOut.isNotEmpty() + override val hasStdErr: Boolean = stdErr != null && stdErr.isNotEmpty() +} + +data class AbortResponseWithMessage( + val result: MimeTypedResult?, + override val stdErr: String? = null, +): Response{ + override val state: ResponseState = ResponseState.Abort + override val stdOut: String? = null + override val hasStdOut: Boolean = false + override val hasStdErr: Boolean = stdErr != null && stdErr.isNotEmpty() +} + +data class ErrorResponseWithMessage( + val result: MimeTypedResult?, + override val stdErr: String? = null, + val errorName: String = "Unknown error", + var errorValue: String = "", + val traceback: List = emptyList(), + val additionalInfo: JsonObject = jsonObject(), +): Response{ + override val state: ResponseState = ResponseState.Error + override val stdOut: String? = null + override val hasStdOut: Boolean = false + override val hasStdErr: Boolean = stdErr != null && stdErr.isNotEmpty() } fun JupyterConnection.Socket.sendOut(msg:Message, stream: JupyterOutType, text: String) { @@ -62,12 +101,19 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup content = jsonObject( "history" to listOf() // not implemented ))) + "interrupt_request" -> { + log.warn("Interruption is not yet supported!") + send(makeReplyMessage(msg, "interrupt_reply", content = msg.content)) + } "shutdown_request" -> { - sendWrapped(msg, makeReplyMessage(msg, "shutdown_reply", content = msg.content)) - Thread.currentThread().interrupt() + send(makeReplyMessage(msg, "shutdown_reply", content = msg.content)) + exitProcess(0) } + + // TODO: This request is deprecated since messaging protocol v.5.1, + // remove it in future versions of kernel "connect_request" -> - sendWrapped(msg, makeReplyMessage(msg, "connection_reply", + sendWrapped(msg, makeReplyMessage(msg, "connect_reply", content = jsonObject(JupyterSockets.values() .map { Pair("${it.name}_port", connection.config.ports[it.ordinal]) }))) "execute_request" -> { @@ -75,16 +121,26 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup val count = executionCount.getAndIncrement() val startedTime = ISO8601DateNow + fun displayHandler(value: Any) { + val res = value.toMimeTypedResult() + connection.iopub.send(makeReplyMessage(msg, + "display_data", + content = jsonObject( + "data" to res, + "metadata" to jsonObject() + ))) + } + connection.iopub.send(makeReplyMessage(msg, "status", content = jsonObject("execution_state" to "busy"))) val code = msg.content["code"] connection.iopub.send(makeReplyMessage(msg, "execute_input", content = jsonObject( "execution_count" to count, "code" to code))) - val res: ResponseWithMessage = if (isCommand(code.toString())) { + val res: Response = if (isCommand(code.toString())) { runCommand(code.toString(), repl) } else { - connection.evalWithIO (repl?.outputConfig) { - repl?.eval(code.toString(), count.toInt()) + connection.evalWithIO(repl!!.outputConfig, msg) { + repl.eval(code.toString(), ::displayHandler, count.toInt()) } } @@ -95,23 +151,17 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup sendOut(msg, JupyterOutType.STDERR, res.stdErr!!) } - when (res.state) { - ResponseState.Ok -> { + when (res) { + is OkResponseWithMessage -> { if (res.result != null) { + val metadata = if (res.result.isolatedHtml) + jsonObject("text/html" to jsonObject("isolated" to true)) else jsonObject() connection.iopub.send(makeReplyMessage(msg, "execute_result", content = jsonObject( "execution_count" to count, "data" to res.result, - "metadata" to jsonObject() - ))) - } - res.displays.forEach { - connection.iopub.send(makeReplyMessage(msg, - "display_data", - content = jsonObject( - "data" to it, - "metadata" to jsonObject() + "metadata" to metadata ))) } @@ -128,7 +178,19 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup "payload" to listOf(), "user_expressions" to JsonObject()))) } - ResponseState.Error -> { + is ErrorResponseWithMessage -> { + val errorReply = makeReplyMessage(msg, "execute_reply", + content = jsonObject( + "status" to "error", + "execution_count" to count, + "ename" to res.errorName, + "evalue" to res.errorValue, + "traceback" to res.traceback, + "additionalInfo" to res.additionalInfo)) + System.err.println("Sending error: $errorReply") + send(errorReply) + } + is AbortResponseWithMessage -> { val errorReply = makeReplyMessage(msg, "execute_reply", content = jsonObject( "status" to "abort", @@ -147,18 +209,33 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup "complete_request" -> { val code = msg.content["code"].toString() val cursor = msg.content["cursor_pos"] as Int - val result = repl?.complete(code, cursor)?.toJson() - if (result == null) { + if (repl == null) { System.err.println("Repl is not yet initialized on complete request") return } - sendWrapped(msg, makeReplyMessage(msg, "complete_reply", content = result)) + GlobalScope.launch { + repl.complete(code, cursor) { result -> + sendWrapped(msg, makeReplyMessage(msg, "complete_reply", content = result.toJson())) + } + } + } + "list_errors_request" -> { + val code = msg.content["code"].toString() + if (repl == null) { + System.err.println("Repl is not yet initialized on listErrors request") + return + } + GlobalScope.launch { + repl.listErrors(code) { result -> + sendWrapped(msg, makeReplyMessage(msg, "list_errors_reply", content = result.toJson())) + } + } } "is_complete_request" -> { val code = msg.content["code"].toString() val resStr = if (isCommand(code)) "complete" else { val result = try { - val check = repl?.checkComplete(executionCount.get(), code) + val check = repl?.checkComplete(code) when { check == null -> "error: no repl" check.isComplete -> "complete" @@ -175,10 +252,12 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup } } -class CapturingOutputStream(private val stdout: PrintStream, - private val conf: OutputConfig, - private val captureOutput: Boolean, - val onCaptured: (String) -> Unit) : OutputStream() { +class CapturingOutputStream( + private val stdout: PrintStream, + private val conf: OutputConfig, + private val captureOutput: Boolean, + val onCaptured: (String) -> Unit, +) : OutputStream() { private val capturedLines = ByteArrayOutputStream() private val capturedNewLine = ByteArrayOutputStream() private var overallOutputSize = 0 @@ -197,7 +276,7 @@ class CapturingOutputStream(private val stdout: PrintStream, private fun flushIfNeeded(b: Int) { val c = b.toChar() - if (c == '\n' || c == '\r') { + if (c == '\n') { newlineFound = true capturedNewLine.writeTo(capturedLines) capturedNewLine.reset() @@ -248,21 +327,19 @@ class CapturingOutputStream(private val stdout: PrintStream, fun Any.toMimeTypedResult(): MimeTypedResult? = when (this) { is MimeTypedResult -> this is Unit -> null - is DisplayResult -> value.toMimeTypedResult() else -> textResult(this.toString()) } -fun JupyterConnection.evalWithIO(maybeConfig: OutputConfig?, body: () -> EvalResult?): ResponseWithMessage { +fun JupyterConnection.evalWithIO(config: OutputConfig, srcMessage: Message, body: () -> EvalResult?): Response { val out = System.out val err = System.err - val config = maybeConfig ?: OutputConfig() fun getCapturingStream(stream: PrintStream, outType: JupyterOutType, captureOutput: Boolean): CapturingOutputStream { return CapturingOutputStream( stream, config, captureOutput) { text -> - this.iopub.sendOut(contextMessage!!, outType, text) + this.iopub.sendOut(srcMessage, outType, text) } } @@ -278,68 +355,62 @@ fun JupyterConnection.evalWithIO(maybeConfig: OutputConfig?, body: () -> EvalRes return try { val exec = body() if (exec == null) { - ResponseWithMessage(ResponseState.Error, textResult("Error!"), emptyList(), null, "NO REPL!") + AbortResponseWithMessage(textResult("Error!"), "NO REPL!") } else { forkedOut.flush() forkedError.flush() try { - var result: MimeTypedResult? = null - val displays = exec.displayValues.mapNotNull { it.toMimeTypedResult() }.toMutableList() - if (exec.resultValue is DisplayResult) { - val resultDisplay = exec.resultValue.value.toMimeTypedResult() - if (resultDisplay != null) - displays += resultDisplay - } else result = exec.resultValue?.toMimeTypedResult() - ResponseWithMessage(ResponseState.Ok, result, displays, null, null) + val result = exec.resultValue?.toMimeTypedResult() + OkResponseWithMessage(result) } catch (e: Exception) { - ResponseWithMessage(ResponseState.Error, textResult("Error!"), emptyList(), null, - "error: Unable to convert result to a string: $e") + AbortResponseWithMessage(textResult("Error!"), "error: Unable to convert result to a string: $e") } } } catch (ex: ReplCompilerException) { forkedOut.flush() forkedError.flush() - // handle runtime vs. compile time and send back correct format of response, now we just send text - /* - { - 'status' : 'error', - 'ename' : str, # Exception name, as a string - 'evalue' : str, # Exception value, as a string - 'traceback' : list(str), # traceback frames as strings - } - */ - ResponseWithMessage(ResponseState.Error, textResult("Error!"), emptyList(), null, - ex.errorResult.message) + val firstDiagnostic = ex.firstDiagnostics + val additionalInfo = firstDiagnostic?.location?.let { + val errorMessage = firstDiagnostic.message + jsonObject("lineStart" to it.start.line, "colStart" to it.start.col, + "lineEnd" to (it.end?.line ?: -1), "colEnd" to (it.end?.col ?: -1), + "message" to errorMessage, + "path" to firstDiagnostic.sourcePath.orEmpty()) + } ?: jsonObject() + + ErrorResponseWithMessage( + textResult("Error!"), + ex.message, + ex.javaClass.canonicalName, + ex.message ?: "", + ex.stackTrace.map { it.toString() }, + additionalInfo) } catch (ex: ReplEvalRuntimeException) { forkedOut.flush() - // handle runtime vs. compile time and send back correct format of response, now we just send text - /* - { - 'status' : 'error', - 'ename' : str, # Exception name, as a string - 'evalue' : str, # Exception value, as a string - 'traceback' : list(str), # traceback frames as strings - } - */ val stdErr = StringBuilder() with(stdErr) { - val cause = ex.errorResult.cause - if (cause == null) appendln(ex.errorResult.message) + val cause = ex.cause + if (cause == null) appendLine(ex.message) else { when (cause) { - is InvocationTargetException -> appendln(cause.targetException.toString()) - else -> appendln(cause.toString()) + is InvocationTargetException -> appendLine(cause.targetException.toString()) + else -> appendLine(cause.toString()) } cause.stackTrace?.also { for (s in it) - appendln(s) + appendLine(s) } } } - ResponseWithMessage(ResponseState.Error, textResult("Error!"), emptyList(), null, stdErr.toString()) + ErrorResponseWithMessage( + textResult("Error!"), + stdErr.toString(), + ex.javaClass.canonicalName, + ex.message ?: "", + ex.stackTrace.map { it.toString() }) } } finally { forkedOut.close() diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt index 536fa7531..240aa5cdf 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl.kt @@ -1,62 +1,174 @@ package org.jetbrains.kotlin.jupyter import jupyter.kotlin.* -import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation -import org.jetbrains.kotlin.cli.common.repl.* +import jupyter.kotlin.KotlinContext +import jupyter.kotlin.KotlinReceiver +import kotlinx.coroutines.runBlocking import org.jetbrains.kotlin.config.KotlinCompilerVersion import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult import org.jetbrains.kotlin.jupyter.repl.completion.KotlinCompleter -import jupyter.kotlin.completion.KotlinContext -import jupyter.kotlin.completion.KotlinReceiver +import org.jetbrains.kotlin.jupyter.repl.completion.ListErrorsResult +import org.jetbrains.kotlin.jupyter.repl.completion.SourceCodeImpl import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater import org.jetbrains.kotlin.jupyter.repl.spark.ClassWriter +import org.jetbrains.kotlin.scripting.ide_services.compiler.KJvmReplCompilerWithIdeServices import java.io.File import java.net.URLClassLoader -import java.util.concurrent.locks.ReentrantReadWriteLock +import java.util.* import kotlin.script.dependencies.ScriptContents import kotlin.script.experimental.api.* import kotlin.script.experimental.host.withDefaultsFrom import kotlin.script.experimental.jvm.* -import kotlin.script.experimental.jvmhost.repl.JvmReplCompiler -import kotlin.script.experimental.jvmhost.repl.JvmReplEvaluator +import kotlin.script.experimental.jvm.util.isError +import kotlin.script.experimental.jvm.util.isIncomplete +import kotlin.script.experimental.jvm.util.toSourceCodePosition -data class EvalResult(val resultValue: Any?, val displayValues: List = emptyList()) +data class EvalResult(val resultValue: Any?) -data class CheckResult(val codeLine: LineId, val isComplete: Boolean = true) +data class CheckResult(val isComplete: Boolean = true) open class ReplException(message: String, cause: Throwable? = null) : Exception(message, cause) -class ReplEvalRuntimeException(val errorResult: ReplEvalResult.Error.Runtime) : ReplException(errorResult.message, errorResult.cause) +class ReplEvalRuntimeException(message: String, cause: Throwable? = null) : ReplException(message, cause) -class ReplCompilerException(val errorResult: ReplCompileResult.Error) : ReplException(errorResult.message) { - constructor (checkResult: ReplCheckResult.Error) : this(ReplCompileResult.Error(checkResult.message, checkResult.location)) - constructor (incompleteResult: ReplCompileResult.Incomplete) : this(ReplCompileResult.Error("Incomplete Code", null)) - constructor (checkResult: ReplEvalResult.Error.CompileTime) : this(ReplCompileResult.Error(checkResult.message, checkResult.location)) - constructor (incompleteResult: ReplEvalResult.Incomplete) : this(ReplCompileResult.Error("Incomplete Code", null)) - constructor (historyMismatchResult: ReplEvalResult.HistoryMismatch) : this(ReplCompileResult.Error("History Mismatch", CompilerMessageLocation.create(null, historyMismatchResult.lineNo, 0, null))) - constructor (message: String) : this(ReplCompileResult.Error(message, null)) +class ReplCompilerException(val errorResult: ResultWithDiagnostics.Failure? = null, message: String? = null) + : ReplException(message ?: errorResult?.getErrors()?.message ?: "") { + + val firstDiagnostics = errorResult?.reports?.firstOrNull { + it.severity == ScriptDiagnostic.Severity.ERROR || it.severity == ScriptDiagnostic.Severity.FATAL + } + + constructor(message: String): this(null, message) +} + +enum class ExecutedCodeLogging { + Off, + All, + Generated +} + +interface ReplOptions { + var trackClasspath: Boolean + + var executedCodeLogging: ExecutedCodeLogging + + var writeCompiledClasses: Boolean + + var outputConfig: OutputConfig +} + +typealias MethodName = String +typealias TypeName = String +typealias Code = String +typealias FieldName = String + +interface ReplForJupyter { + fun eval(code: Code, displayHandler: ((Any) -> Unit)? = null, jupyterId: Int = -1): EvalResult + + fun checkComplete(code: Code): CheckResult + + suspend fun complete(code: String, cursor: Int, callback: (CompletionResult) -> Unit) + + suspend fun listErrors(code: String, callback: (ListErrorsResult) -> Unit) + + val currentClasspath: Collection get + + val resolverConfig: ResolverConfig? get + + var outputConfig: OutputConfig get } -class ReplForJupyter(val scriptClasspath: List = emptyList(), - val config: ResolverConfig? = null) { +class ReplForJupyterImpl(val scriptClasspath: List = emptyList(), + override val resolverConfig: ResolverConfig? = null, vararg scriptReceivers: Any) : ReplForJupyter, ReplOptions, KotlinKernelHost { + + var outputConfigImpl = OutputConfig() + + override var outputConfig + get() = outputConfigImpl + set(value) { + // reuse output config instance, because it is already passed to CapturingOutputStream and stream parameters should be updated immediately + outputConfigImpl.update(value) + } + + override var trackClasspath: Boolean = false - val outputConfig = OutputConfig() + override var executedCodeLogging: ExecutedCodeLogging = ExecutedCodeLogging.Off - private val resolver = JupyterScriptDependenciesResolver(config) + var classWriter: ClassWriter? = null - private val renderers = config?.let { - it.libraries.asyncLet { - it.flatMap { it.value.renderers }.map { it.className to it }.toMap() + override var writeCompiledClasses: Boolean + get() = classWriter != null + set(value) { + if (!value) classWriter = null + else { + val cw = ClassWriter() + System.setProperty("spark.repl.class.outputDir", cw.outputDir.toString()) + classWriter = cw + } } + + private val resolver = JupyterScriptDependenciesResolver(resolverConfig) + + private val typeRenderers = mutableMapOf() + + private val initCellCodes = mutableListOf() + + private fun renderResult(value: Any?, resultField: Pair?): Any? { + if (value == null || resultField == null) return null + val code = typeRenderers[value.javaClass.canonicalName]?.replace("\$it", resultField.first) + ?: return value + val result = doEval(code) + return renderResult(result.value, result.resultField) } - private val includedLibraries = mutableSetOf() + data class PreprocessingResult(val code: Code, val initCodes: List, val initCellCodes: List, val typeRenderers: List) + + fun preprocessCode(code: String): PreprocessingResult { + + val processedMagics = magics.processMagics(code) + + val initCodes = mutableListOf() + val initCellCodes = mutableListOf() + val typeRenderers = mutableListOf() + val typeConverters = mutableListOf() + val annotations = mutableListOf() + + processedMagics.libraries.forEach { + val builder = StringBuilder() + it.repositories.forEach { builder.appendLine("@file:Repository(\"$it\")") } + it.dependencies.forEach { builder.appendLine("@file:DependsOn(\"$it\")") } + it.imports.forEach { builder.appendLine("import $it") } + if (builder.isNotBlank()) + initCodes.add(builder.toString()) + typeRenderers.addAll(it.renderers) + typeConverters.addAll(it.converters) + annotations.addAll(it.annotations) + it.init.forEach { + + // Library init code may contain other magics, so we process them recursively + val preprocessed = preprocessCode(it) + initCodes.addAll(preprocessed.initCodes) + typeRenderers.addAll(preprocessed.typeRenderers) + initCellCodes.addAll(preprocessed.initCellCodes) + if (preprocessed.code.isNotBlank()) + initCodes.add(preprocessed.code) + } + } + + val declarations = (typeConverters.map { typeProvidersProcessor.register(it) } + annotations.map { annotationsProcessor.register(it) }) + .joinToString("\n") + if (declarations.isNotBlank()) { + initCodes.add(declarations) + } + + return PreprocessingResult(processedMagics.code, initCodes, initCellCodes, typeRenderers) + } - fun preprocessCode(code: String) = processMagics(this, code) + private val ctx = KotlinContext() - private val receiver = KotlinReceiver() + private val receivers: List = listOf(KotlinReceiver(ctx)) + scriptReceivers - val librariesCodeGenerator = LibrariesProcessor() + val magics = MagicsProcessor(this, LibrariesProcessor(resolverConfig?.libraries)) private fun configureMavenDepsOnAnnotations(context: ScriptConfigurationRefinementContext): ResultWithDiagnostics { val annotations = context.collectedData?.get(ScriptCollectedData.foundAnnotations)?.takeIf { it.isNotEmpty() } @@ -91,10 +203,8 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), onAnnotations(DependsOn::class, Repository::class, handler = { configureMavenDepsOnAnnotations(it) }) } - val kt = KotlinType(receiver.javaClass.canonicalName) - implicitReceivers.invoke(listOf(kt)) + implicitReceivers.invoke(receivers.map { KotlinType(it.javaClass.canonicalName) }) - log.info("Classpath for compiler options: none") compilerOptions.invoke(listOf("-jvm-target", "1.8")) } } @@ -105,12 +215,11 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), ?.flatMap { it.classpath } .orEmpty() - val currentClasspath = mutableSetOf().also { it.addAll(compilerConfiguration.classpath.map { it.canonicalPath }) } + override val currentClasspath = mutableSetOf().also { it.addAll(compilerConfiguration.classpath.map { it.canonicalPath }) } private class FilteringClassLoader(parent: ClassLoader, val includeFilter: (String) -> Boolean) : ClassLoader(parent) { override fun loadClass(name: String?, resolve: Boolean): Class<*> { - var c: Class<*>? = null - c = if (name != null && includeFilter(name)) + val c = if (name != null && includeFilter(name)) parent.loadClass(name) else parent.parent.loadClass(name) if (resolve) @@ -120,7 +229,7 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), } private val evaluatorConfiguration = ScriptEvaluationConfiguration { - implicitReceivers.invoke(receiver) + implicitReceivers.invoke(v = receivers) jvm { val filteringClassLoader = FilteringClassLoader(ClassLoader.getSystemClassLoader()) { it.startsWith("jupyter.kotlin.") || it.startsWith("kotlin.") || (it.startsWith("org.jetbrains.kotlin.") && !it.startsWith("org.jetbrains.kotlin.jupyter.")) @@ -133,163 +242,285 @@ class ReplForJupyter(val scriptClasspath: List = emptyList(), private var executionCounter = 0 - private val compiler: ReplCompiler by lazy { - JvmReplCompiler(compilerConfiguration) + private val compiler: KJvmReplCompilerWithIdeServices by lazy { + KJvmReplCompilerWithIdeServices() } - private val evaluator: ReplEvaluator by lazy { - JvmReplEvaluator(evaluatorConfiguration) + private val evaluator: BasicJvmReplEvaluator by lazy { + BasicJvmReplEvaluator() } - private val stateLock = ReentrantReadWriteLock() + private val completer = KotlinCompleter() - private val compilerState = compiler.createState(stateLock) + private val contextUpdater = ContextUpdater(ctx, evaluator) - private val evaluatorState = evaluator.createState(stateLock) + private val typeProvidersProcessor: TypeProvidersProcessor = TypeProvidersProcessorImpl(contextUpdater) - private val state = AggregatedReplStageState(compilerState, evaluatorState, stateLock) + private val annotationsProcessor: AnnotationsProcessor = AnnotationsProcessorImpl(contextUpdater) - private val ctx = KotlinContext() + private var currentDisplayHandler: ((Any) -> Unit)? = null - private val contextUpdater = ContextUpdater(state, ctx.vars, ctx.functions) + private val scheduledExecutions = LinkedList() - private val completer = KotlinCompleter(ctx) + override fun checkComplete(code: String): CheckResult { + val id = executionCounter++ + val codeLine = SourceCodeImpl(id, code) + val result = runBlocking { compiler.analyze(codeLine, 0.toSourceCodePosition(codeLine), compilerConfiguration) } + return when { + result.isIncomplete() -> CheckResult(false) + result.isError() -> throw ReplException(result.getErrors().message) + else -> CheckResult(true) + } + } - var trackClasspath: Boolean = false + init { + // TODO: to be removed after investigation of https://github.com/kotlin/kotlin-jupyter/issues/24 + doEval("1") + } - var trackExecutedCode: Boolean = false + private fun executeInitCellCode() = initCellCodes.forEach(::evalNoReturn) - var classWriter: ClassWriter? = null + private fun executeInitCode(p: PreprocessingResult) = p.initCodes.forEach(::evalNoReturn) - fun checkComplete(executionNumber: Long, code: String): CheckResult { - val codeLine = ReplCodeLine(executionNumber.toInt(), 0, code) - return when (val result = compiler.check(compilerState, codeLine)) { - is ReplCheckResult.Error -> throw ReplCompilerException(result) - is ReplCheckResult.Ok -> CheckResult(LineId(codeLine), true) - is ReplCheckResult.Incomplete -> CheckResult(LineId(codeLine), false) - else -> throw IllegalStateException("Unknown check result type ${result}") + private fun executeScheduledCode() { + while (scheduledExecutions.isNotEmpty()) { + val code = scheduledExecutions.pop() + if (executedCodeLogging == ExecutedCodeLogging.Generated) + println(code) + evalNoReturn(code) } } - init { - // TODO: to be removed after investigation of https://github.com/kotlin/kotlin-jupyter/issues/24 - doEval("1") + private fun processVariablesConversion() { + var iteration = 0 + do { + if (iteration++ > 10) { + log.error("Execution loop in type providers processing") + break + } + val codes = typeProvidersProcessor.process() + codes.forEach { + if (executedCodeLogging == ExecutedCodeLogging.Generated) + println(it) + evalNoReturn(it) + } + } while (codes.isNotEmpty()) + } + + private fun processAnnotations(replLine: Any?) { + if (replLine == null) return + log.catchAll { + annotationsProcessor.process(replLine) + }?.forEach { + if (executedCodeLogging == ExecutedCodeLogging.Generated) + println(it) + evalNoReturn(it) + } + } + + fun registerNewLibraries(p: PreprocessingResult) { + p.initCellCodes.filter { !initCellCodes.contains(it) }.let(initCellCodes::addAll) + typeRenderers.putAll(p.typeRenderers.map { it.className to it.code }) } - fun eval(code: String, jupyterId: Int = -1): EvalResult { + private fun lastReplLine() = evaluator.lastEvaluatedSnippet?.get()?.result?.scriptClass + + override fun eval(code: String, displayHandler: ((Any) -> Unit)?, jupyterId: Int): EvalResult { synchronized(this) { try { - val displays = mutableListOf() - - val initCell = includedLibraries.flatMap { it.initCell }.joinToString(separator = "\n") - if (initCell.isNotBlank()) - (doEval(initCell).value as? DisplayResult)?.let(displays::add) + currentDisplayHandler = displayHandler - val processedCode = preprocessCode(code) + executeInitCellCode() - val newLibraries = librariesCodeGenerator.getProcessedLibraries() + val preprocessed = preprocessCode(code) - newLibraries.forEach { - (doEval(it.code).value as? DisplayResult)?.let(displays::add) - } + executeInitCode(preprocessed) var result: Any? = null - var replId = -1 + var resultField: Pair? = null + var replLine: Any? = null - if (processedCode.isNotBlank()) { - val e = doEval(processedCode) - result = e.value - replId = e.replId + if (preprocessed.code.isNotBlank()) { + doEval(preprocessed.code).let { + result = it.value + resultField = it.resultField + } + replLine = lastReplLine() } - // on successful execution add all new libraries to the set - includedLibraries.addAll(newLibraries.map { it.library }) + processAnnotations(replLine) - if (jupyterId >= 0) { - while (ReplOutputs.count() <= jupyterId) ReplOutputs.add(null) - ReplOutputs[jupyterId] = result - } + executeScheduledCode() - val resolvedClasspath = resolver.popAddedClasspath().map { it.canonicalPath } - if (resolvedClasspath.isNotEmpty()) { - - val newClasspath = resolvedClasspath.filter { !currentClasspath.contains(it) } - val oldClasspath = resolvedClasspath.filter { currentClasspath.contains(it) } - currentClasspath.addAll(newClasspath) - if (trackClasspath) { - val sb = StringBuilder() - if (newClasspath.count() > 0) { - sb.appendln("${newClasspath.count()} new paths were added to classpath:") - newClasspath.sortedBy { it }.forEach { sb.appendln(it) } - } - if (oldClasspath.count() > 0) { - sb.appendln("${oldClasspath.count()} resolved paths were already in classpath:") - oldClasspath.sortedBy { it }.forEach { sb.appendln(it) } - } - sb.appendln("Current classpath size: ${currentClasspath.count()}") - displays.add(sb.toString()) - } - } + registerNewLibraries(preprocessed) + + processVariablesConversion() + + executeScheduledCode() + + updateOutputList(jupyterId, result) + + updateClasspath() + + result = renderResult(result, resultField) + + return EvalResult(result) - if (result != null && renderers != null) { - val resultType = result.javaClass.canonicalName - renderers.awaitBlocking()[resultType]?.let { - it.displayCode?.let { - doEval(it.replace("\$it", "res$replId")).value?.let(displays::add) - } - result = if (it.resultCode == null || it.resultCode.trim().isBlank()) "" - else doEval(it.resultCode.replace("\$it", "res$replId")).value - } - if (result is DisplayResult) { - displays.add(result as Any) - result = "" - } - } - return EvalResult(result, displays) } finally { - librariesCodeGenerator.getProcessedLibraries() + currentDisplayHandler = null + scheduledExecutions.clear() } } } - fun complete(code: String, cursor: Int): CompletionResult = completer.complete(code, cursor) + private fun updateOutputList(jupyterId: Int, result: Any?) { + if (jupyterId >= 0) { + while (ReplOutputs.count() <= jupyterId) ReplOutputs.add(null) + ReplOutputs[jupyterId] = result + } + } + + private fun updateClasspath() { + val resolvedClasspath = resolver.popAddedClasspath().map { it.canonicalPath } + if (resolvedClasspath.isNotEmpty()) { + + val newClasspath = resolvedClasspath.filter { !currentClasspath.contains(it) } + val oldClasspath = resolvedClasspath.filter { currentClasspath.contains(it) } + currentClasspath.addAll(newClasspath) + if (trackClasspath) { + val sb = StringBuilder() + if (newClasspath.count() > 0) { + sb.appendLine("${newClasspath.count()} new paths were added to classpath:") + newClasspath.sortedBy { it }.forEach { sb.appendLine(it) } + } + if (oldClasspath.count() > 0) { + sb.appendLine("${oldClasspath.count()} resolved paths were already in classpath:") + oldClasspath.sortedBy { it }.forEach { sb.appendLine(it) } + } + sb.appendLine("Current classpath size: ${currentClasspath.count()}") + println(sb.toString()) + } + } + } - private data class InternalEvalResult(val value: Any?, val replId: Int) + private val completionQueue = LockQueue() + override suspend fun complete(code: String, cursor: Int, callback: (CompletionResult) -> Unit) = doWithLock(CompletionArgs(code, cursor, callback), completionQueue, CompletionResult.Empty(code, cursor)) { + //val preprocessed = preprocessCode(code) + completer.complete(compiler, compilerConfiguration, code, executionCounter++, cursor) + } + + private val listErrorsQueue = LockQueue() + override suspend fun listErrors(code: String, callback: (ListErrorsResult) -> Unit) = doWithLock(ListErrorsArgs(code, callback), listErrorsQueue, ListErrorsResult(code)) { + //val preprocessed = preprocessCode(code) + val codeLine = SourceCodeImpl(executionCounter++, code) + val errorsList = runBlocking { compiler.analyze(codeLine, 0.toSourceCodePosition(codeLine), compilerConfiguration) } + ListErrorsResult(code, errorsList.valueOrThrow()) + } + + private fun > doWithLock(args: Args, queue: LockQueue, default: T, action: (Args) -> T) { + queue.add(args) + + val result = synchronized(this) { + val lastArgs = queue.get() + if (lastArgs != args) + default + else + action(args) + } + args.callback(result) + } + + private fun evalNoReturn(code: String) { + doEval(code) + processAnnotations(lastReplLine()) + } + + private data class InternalEvalResult(val value: Any?, val resultField: Pair?) + + private interface LockQueueArgs { + val callback: (T) -> Unit + } + + private data class CompletionArgs(val code: String, val cursor: Int, override val callback: (CompletionResult) -> Unit) : LockQueueArgs + private data class ListErrorsArgs(val code: String, override val callback: (ListErrorsResult) -> Unit) : LockQueueArgs + + private class LockQueue> { + private var args: Args? = null + + fun add(args: Args) { + synchronized(this) { + this.args = args + } + } + + fun get(): Args { + return args!! + } + } private fun doEval(code: String): InternalEvalResult { - if (trackExecutedCode) - println("Executing:\n$code\n") + if (executedCodeLogging == ExecutedCodeLogging.All) + println(code) val id = executionCounter++ - val codeLine = ReplCodeLine(id, 0, code) - when (val compileResult = compiler.compile(compilerState, codeLine)) { - is ReplCompileResult.CompiledClasses -> { - classWriter?.writeClasses(compileResult) - val result = evaluator.eval(evaluatorState, compileResult) + val codeLine = SourceCodeImpl(id, code) + when (val compileResultWithDiagnostics = runBlocking { compiler.compile(codeLine, compilerConfiguration) }) { + is ResultWithDiagnostics.Success -> { + val compileResult = compileResultWithDiagnostics.value + classWriter?.writeClasses(codeLine, compileResult.get()) + val repl = this + val currentEvalConfig = ScriptEvaluationConfiguration(evaluatorConfiguration) { + constructorArgs.invoke(repl as KotlinKernelHost) + } + val resultWithDiagnostics = runBlocking { evaluator.eval(compileResult, currentEvalConfig) } contextUpdater.update() - return when (result) { - is ReplEvalResult.Error.CompileTime -> throw ReplCompilerException(result) - is ReplEvalResult.Error.Runtime -> throw ReplEvalRuntimeException(result) - is ReplEvalResult.Incomplete -> throw ReplCompilerException(result) - is ReplEvalResult.HistoryMismatch -> throw ReplCompilerException(result) - is ReplEvalResult.UnitResult -> { - InternalEvalResult(Unit, id) + + when(resultWithDiagnostics) { + is ResultWithDiagnostics.Success -> { + val pureResult = resultWithDiagnostics.value.get() + return when (val resultValue = pureResult.result) { + is ResultValue.Error -> throw ReplEvalRuntimeException(resultValue.error.message.orEmpty(), resultValue.error) + is ResultValue.Unit -> { + InternalEvalResult(Unit, null) + } + is ResultValue.Value -> { + InternalEvalResult(resultValue.value, pureResult.compiledSnippet.resultField) + } + is ResultValue.NotEvaluated -> { + throw ReplEvalRuntimeException(buildString { + val cause = resultWithDiagnostics.reports.firstOrNull()?.exception + val stackTrace = cause?.stackTrace ?: emptyArray() + append("This snippet was not evaluated: ") + appendLine(cause.toString()) + for (s in stackTrace) + appendLine(s) + }) + } + else -> throw IllegalStateException("Unknown eval result type ${this}") + } } - is ReplEvalResult.ValueResult -> { - InternalEvalResult(result.value, id) + is ResultWithDiagnostics.Failure -> { + throw ReplCompilerException(resultWithDiagnostics) } - else -> throw IllegalStateException("Unknown eval result type ${this}") + else -> throw IllegalStateException("Unknown result") } + } - is ReplCompileResult.Error -> throw ReplCompilerException(compileResult) - is ReplCompileResult.Incomplete -> throw ReplCompilerException(compileResult) + is ResultWithDiagnostics.Failure -> throw ReplCompilerException(compileResultWithDiagnostics) } } init { log.info("Starting kotlin REPL engine. Compiler version: ${KotlinCompilerVersion.VERSION}") log.info("Classpath used in script: ${scriptClasspath}") - receiver.kc = ctx + } + + override fun display(value: Any) { + currentDisplayHandler?.invoke(value) + } + + override fun scheduleExecution(code: String) { + scheduledExecutions.add(code) } } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinCompleter.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinCompleter.kt index 6052e0e3d..1d70ce73d 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinCompleter.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinCompleter.kt @@ -1,12 +1,13 @@ package org.jetbrains.kotlin.jupyter.repl.completion import com.beust.klaxon.JsonObject +import kotlinx.coroutines.runBlocking +import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.jupyter.jsonObject -import jupyter.kotlin.completion.KotlinContext -import jupyter.kotlin.completion.KotlinReflectUtil.shorten -import java.util.TreeMap import java.io.PrintWriter import java.io.StringWriter +import kotlin.script.experimental.api.* +import kotlin.script.experimental.jvm.util.toSourceCodePosition enum class CompletionStatus(private val value: String) { OK("ok"), @@ -17,100 +18,135 @@ enum class CompletionStatus(private val value: String) { } } +data class CompletionTokenBounds(val start: Int, val end: Int) + abstract class CompletionResult( - val status: CompletionStatus + private val status: CompletionStatus ) { open fun toJson(): JsonObject { return jsonObject("status" to status.toString()) } -} -data class CompletionTokenBounds(val start: Int, val end: Int) + open class Success( + private val matches: List, + private val bounds: CompletionTokenBounds, + private val metadata: List, + private val text: String, + private val cursor: Int + ): CompletionResult(CompletionStatus.OK) { + init { + assert(matches.size == metadata.size) + } -class CompletionResultSuccess( - val matches: List, - val bounds: CompletionTokenBounds, - val metadata: Map -): CompletionResult(CompletionStatus.OK) { - override fun toJson(): JsonObject { - val res = super.toJson() - res["matches"] = matches - res["cursor_start"] = bounds.start - res["cursor_end"] = bounds.end - res["metadata"] = mapOf("_jupyter_types_experimental" to metadata.map { - mapOf( - "text" to it.key, - "type" to it.value, - "start" to bounds.start, - "end" to bounds.end + override fun toJson(): JsonObject { + val res = super.toJson() + res["matches"] = matches + res["cursor_start"] = bounds.start + res["cursor_end"] = bounds.end + res["metadata"] = mapOf( + "_jupyter_types_experimental" to metadata.map { + mapOf( + "text" to it.text, + "type" to it.tail, + "start" to bounds.start, + "end" to bounds.end + ) + }, + "_jupyter_extended_metadata" to metadata.map { + mapOf( + "text" to it.text, + "displayText" to it.displayText, + "icon" to it.icon, + "tail" to it.tail + ) + } ) - }) - return res + res["paragraph"] = mapOf( + "cursor" to cursor, + "text" to text + ) + return res + } + + @TestOnly + fun sortedMatches(): List = matches.sorted() } -} -class CompletionResultError( - val errorName: String, - val errorValue: String, - val traceBack: String -): CompletionResult(CompletionStatus.ERROR) { - override fun toJson(): JsonObject { - val res = super.toJson() - res["ename"] = errorName - res["evalue"] = errorValue - res["traceback"] = traceBack - return res + class Empty( + text: String, cursor: Int + ): CompletionResult.Success(emptyList(), CompletionTokenBounds(cursor, cursor), emptyList(), text, cursor) + + class Error( + private val errorName: String, + private val errorValue: String, + private val traceBack: String + ): CompletionResult(CompletionStatus.ERROR) { + override fun toJson(): JsonObject { + val res = super.toJson() + res["ename"] = errorName + res["evalue"] = errorValue + res["traceback"] = traceBack + return res + } } } +data class ListErrorsResult(val code: String, val errors: Sequence = emptySequence()) { + fun toJson(): JsonObject { + return jsonObject("code" to code, + "errors" to errors.map { + val er = jsonObject( + "message" to it.message, + "severity" to it.severity.name + ) -class KotlinCompleter(private val ctx: KotlinContext) { - fun complete(buf: String, cursor: Int): CompletionResult { - try { - val bounds = getTokenBounds(buf, cursor) - val token = buf.substring(bounds.start, bounds.end) + val loc = it.location + if (loc != null) { + val start = loc.start + val end = loc.end + er["start"] = jsonObject("line" to start.line, "col" to start.col) + if (end != null) + er["end"] = jsonObject("line" to end.line, "col" to end.col) + } + er + }.toList()) + } +} - val tokens = TreeMap() - val tokensFilter = { t: String -> t.startsWith(token) } - tokens.putAll(keywords.filter { entry -> tokensFilter(entry.key) }) +internal class SourceCodeImpl(number: Int, override val text: String) : SourceCode { + override val name: String? = "Line_$number" + override val locationId: String? = "location_$number" +} - tokens.putAll(ctx.getVarsList().asSequence() - .filter { tokensFilter(it.name) } - .map { it.name to shorten(it.type) }) +class KotlinCompleter { + fun complete(compiler: ReplCompleter, configuration: ScriptCompilationConfiguration, code: String, id: Int, cursor: Int): CompletionResult { + return try { + val codeLine = SourceCodeImpl(id, code) + val completionResult = runBlocking { compiler.complete(codeLine, cursor.toSourceCodePosition(codeLine), configuration) } - tokens.putAll(ctx.getFunctionsList().asSequence() - .filter { tokensFilter(it.name) } - .map { it.name to it.toString(true) }) + completionResult.valueOrNull()?.toList()?.let { completionList -> + val bounds = getTokenBounds(code, cursor) + CompletionResult.Success(completionList.map { it.text }, bounds, completionList, code, cursor) + } ?: CompletionResult.Empty(code, cursor) - return CompletionResultSuccess(tokens.keys.toList(), bounds, tokens) } catch (e: Exception) { val sw = StringWriter() e.printStackTrace(PrintWriter(sw)) - return CompletionResultError(e.javaClass.simpleName, e.message ?: "", sw.toString()) + CompletionResult.Error(e.javaClass.simpleName, e.message ?: "", sw.toString()) } } companion object { - private val keywords = KotlinKeywords.KEYWORDS.asSequence().map { it to "keyword" }.toMap() - fun getTokenBounds(buf: String, cursor: Int): CompletionTokenBounds { require(cursor <= buf.length) { "Position $cursor does not exist in code snippet <$buf>" } val startSubstring = buf.substring(0, cursor) - val endSubstring = buf.substring(cursor) - val filter = {c: Char -> !c.isLetterOrDigit()} + val filter = {c: Char -> !c.isLetterOrDigit() && c != '_'} val start = startSubstring.indexOfLast(filter) + 1 - var end = endSubstring.indexOfFirst(filter) - end = if (end == -1) { - buf.length - } else { - end + startSubstring.length - } - - return CompletionTokenBounds(start, end) + return CompletionTokenBounds(start, cursor) } } } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinKeywords.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinKeywords.kt deleted file mode 100644 index 7a4954f60..000000000 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/completion/KotlinKeywords.kt +++ /dev/null @@ -1,84 +0,0 @@ -package org.jetbrains.kotlin.jupyter.repl.completion - -object KotlinKeywords { - /** - * List of Kotlin keywords for completion. - */ - val KEYWORDS: List = listOf( - "as", - "as?", - "break", - "class", - "continue", - "do", - "else", - "false", - "for", - "fun", - "if", - "in", - "interface", - "is", - "null", - "object", - "package", - "return", - "super", - "this", - "throw", - "true", - "try", - "typealias", - "typeof", - "val", - "var", - "when", - "while", - "by", - "catch", - "constructor", - "delegate", - "dynamic", - "field", - "file", - "finally", - "get", - "import", - "init", - "param", - "property", - "receiver", - "set", - "setparam", - "where", - "actual", - "abstract", - "annotation", - "companion", - "const", - "crossinline", - "data", - "enum", - "expect", - "external", - "final", - "infix", - "inline", - "inner", - "internal", - "lateinit", - "noinline", - "open", - "operator", - "out", - "override", - "private", - "protected", - "public", - "reified", - "sealed", - "suspend", - "tailrec", - "vararg" - ) -} diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/ContextUpdater.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/ContextUpdater.kt index 3100bdbe8..5faf708fe 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/ContextUpdater.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/ContextUpdater.kt @@ -1,33 +1,27 @@ package org.jetbrains.kotlin.jupyter.repl.reflect -import jupyter.kotlin.completion.KotlinFunctionInfo -import jupyter.kotlin.completion.KotlinVariableInfo -import org.jetbrains.kotlin.cli.common.repl.AggregatedReplStageState -import org.jetbrains.kotlin.cli.common.repl.ReplHistoryRecord +import jupyter.kotlin.KotlinContext +import jupyter.kotlin.KotlinFunctionInfo +import jupyter.kotlin.KotlinVariableInfo +import org.jetbrains.kotlin.jupyter.instances import org.slf4j.LoggerFactory import java.lang.reflect.Field import java.util.* -import java.util.stream.Collectors import kotlin.reflect.jvm.kotlinFunction import kotlin.reflect.jvm.kotlinProperty +import kotlin.script.experimental.jvm.BasicJvmReplEvaluator /** * ContextUpdater updates current user-defined functions and variables * to use in completion and KotlinContext. */ -class ContextUpdater(private val state: AggregatedReplStageState<*, *>, - private val vars: MutableMap, - private val functions: MutableSet) { - - private val lines: List - get() = state.history - .mapNotNull { this.getLineFromRecord(it) } - .asReversed() +class ContextUpdater(val context: KotlinContext, private val evaluator: BasicJvmReplEvaluator) { fun update() { try { - val lines = lines + val lastSnippet = evaluator.lastEvaluatedSnippet + val lines = lastSnippet.instances() refreshVariables(lines) refreshMethods(lines) } catch (e: ReflectiveOperationException) { @@ -39,7 +33,7 @@ class ContextUpdater(private val state: AggregatedReplStageState<*, *>, } private fun refreshMethods(lines: List) { - functions.clear() + context.functions.clear() for (line in lines) { val methods = line.javaClass.methods for (method in methods) { @@ -47,16 +41,11 @@ class ContextUpdater(private val state: AggregatedReplStageState<*, *>, continue } val function = method.kotlinFunction ?: continue - functions.add(KotlinFunctionInfo(function)) + context.functions.putIfAbsent(function.name, KotlinFunctionInfo(function, line)) } } } - private fun getLineFromRecord(record: ReplHistoryRecord>): Any? { - val statePair = record.item.second - return (statePair as Pair<*, *>).second - } - @Throws(ReflectiveOperationException::class) private fun getImplicitReceiver(script: Any): Any { val receiverField = script.javaClass.getDeclaredField("\$\$implicitReceiver0") @@ -65,7 +54,7 @@ class ContextUpdater(private val state: AggregatedReplStageState<*, *>, @Throws(ReflectiveOperationException::class) private fun refreshVariables(lines: List) { - vars.clear() + context.vars.clear() if (lines.isNotEmpty()) { val receiver = getImplicitReceiver(lines[0]) findReceiverVariables(receiver) @@ -107,7 +96,7 @@ class ContextUpdater(private val state: AggregatedReplStageState<*, *>, if (!fieldName.contains("script$")) { val descriptor = field.kotlinProperty if (descriptor != null) { - vars.putIfAbsent(fieldName, KotlinVariableInfo(value, descriptor)) + context.vars.putIfAbsent(fieldName, KotlinVariableInfo(value, descriptor, o)) } } } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/History.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/History.kt new file mode 100644 index 000000000..e6e7448c1 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/reflect/History.kt @@ -0,0 +1,8 @@ +package org.jetbrains.kotlin.jupyter.repl.reflect + +import org.jetbrains.kotlin.cli.common.repl.AggregatedReplStageState + +val AggregatedReplStageState<*, *>.lines + get() = history + .mapNotNull { (it.item.second as Pair<*, *>).second } + .asReversed() \ No newline at end of file diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/spark/ClassWriter.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/spark/ClassWriter.kt index f466447f2..694b66e25 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/spark/ClassWriter.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/repl/spark/ClassWriter.kt @@ -1,15 +1,14 @@ package org.jetbrains.kotlin.jupyter.repl.spark -import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult.CompiledClasses -import org.jetbrains.kotlin.scripting.compiler.plugin.impl.KJvmCompiledModuleInMemory import org.slf4j.LoggerFactory import java.io.BufferedOutputStream -import java.io.File import java.io.FileOutputStream import java.io.IOException import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import kotlin.script.experimental.api.SourceCode +import kotlin.script.experimental.jvm.impl.KJvmCompiledModuleInMemory import kotlin.script.experimental.jvm.impl.KJvmCompiledScript @@ -32,28 +31,22 @@ class ClassWriter(_outputDir: String = "") { logger.info("Created ClassWriter with path <$outputDir>") } - fun writeClasses(classes: CompiledClasses) { - for ((filePath, bytes) in classes.classes) { - if (!filePath.contains(File.separator)) { - writeClass(bytes, outputDir.resolve(filePath)) - } - } - writeModuleInMemory(classes) + fun writeClasses(code: SourceCode, classes: KJvmCompiledScript) { + writeModuleInMemory(code, classes) } - private fun writeModuleInMemory(classes: CompiledClasses) { + private fun writeModuleInMemory(code: SourceCode, classes: KJvmCompiledScript) { try { - val compiledScript = classes.data as KJvmCompiledScript<*> - val moduleInMemory = compiledScript.compiledModule as KJvmCompiledModuleInMemory + val moduleInMemory = classes.getCompiledModule() as KJvmCompiledModuleInMemory moduleInMemory.compilerOutputFiles.forEach { (name, bytes) -> if (name.contains("class")) { writeClass(bytes, outputDir.resolve(name)) } } } catch (e: ClassCastException) { - logger.info("Compiled line #" + classes.lineId.no + "has no in-memory modules") + logger.info("Compiled line " + code.name + " has no in-memory modules") } catch (e: NullPointerException) { - logger.info("Compiled line #" + classes.lineId.no + "has no in-memory modules") + logger.info("Compiled line " + code.name + " has no in-memory modules") } } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt index 3aaa421ff..5c91b3cdd 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/resolver.kt @@ -52,7 +52,7 @@ open class JupyterScriptDependenciesResolver(resolverConfig: ResolverConfig?) { val result = runBlocking { resolver.resolve(annotation.value) } when (result) { is ResultWithDiagnostics.Failure -> { - val diagnostics = ScriptDiagnostic("Failed to resolve ${annotation.value}:\n" + result.reports.joinToString("\n") { it.message }) + val diagnostics = ScriptDiagnostic(ScriptDiagnostic.unspecifiedError, "Failed to resolve ${annotation.value}:\n" + result.reports.joinToString("\n") { it.message }) log.warn(diagnostics.message, diagnostics.exception) scriptDiagnostics.add(diagnostics) } @@ -63,7 +63,7 @@ open class JupyterScriptDependenciesResolver(resolverConfig: ResolverConfig?) { } } } catch (e: Exception) { - val diagnostic = ScriptDiagnostic("Unhandled exception during resolve", exception = e) + val diagnostic = ScriptDiagnostic(ScriptDiagnostic.unspecifiedError, "Unhandled exception during resolve", exception = e) log.error(diagnostic.message, e) scriptDiagnostics.add(diagnostic) } diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/typeProviders.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/typeProviders.kt new file mode 100644 index 000000000..6fdb829db --- /dev/null +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/typeProviders.kt @@ -0,0 +1,129 @@ +package org.jetbrains.kotlin.jupyter + +import jupyter.kotlin.KotlinFunctionInfo +import org.jetbrains.kotlin.jupyter.repl.reflect.ContextUpdater +import kotlin.reflect.KMutableProperty +import kotlin.reflect.KProperty +import kotlin.reflect.full.withNullability +import kotlin.reflect.jvm.isAccessible + +interface TypeProvidersProcessor { + + fun register(handler: TypeHandler): Code + + fun process(): List +} + +class TypeProvidersProcessorImpl(private val contextUpdater: ContextUpdater) : TypeProvidersProcessor { + + private val handlers = mutableMapOf() + + private val methodIdMap = mutableMapOf() + + private val codeToMethodMap = mutableMapOf() + + private var nextGeneratedMethodId = 0 + + private class TypeConverterCodes(val declarations: List, val converter: Code, val wildcard: String? = "###") { + + constructor(code: List) : this(code.subList(0, code.size - 1), code.last()) + + val hasWildcard = wildcard != null && (declarations.any { it.contains(wildcard) } || converter.contains(wildcard)) + + val fullCode: String = declarations.joinToString("\n") + "\n" + converter + + override fun toString(): String { + return fullCode + } + + fun withoutDeclarations() = TypeConverterCodes(emptyList(), converter, wildcard) + + fun replaceWildcard(str: String) = if (!hasWildcard) throw Exception() else + TypeConverterCodes(declarations.map { it.replace(wildcard!!, str) }, converter.replace(wildcard!!, str), null) + } + + private fun getMethodName(id: Int) = "___getConverter$id" + + override fun register(handler: TypeHandler): Code { + val instanceArg = "__it" + val propertyArg = "__property" + val body = handler.code.replace("\$it", instanceArg).replace("\$property", propertyArg) + val type = handler.className + val methodId = nextGeneratedMethodId++ + val methodName = getMethodName(methodId) + methodIdMap[type] = methodId + return "fun $methodName($instanceArg : $type, $propertyArg : kotlin.reflect.KProperty<*>) = $body" + } + + private fun toRegex(name: TypeName) = name.split('.').joinToString("\\.").split('*').joinToString(".*").toRegex() + + override fun process(): List { + + if (methodIdMap.isNotEmpty()) { + contextUpdater.update() + handlers.putAll(methodIdMap.map { + toRegex(it.key) to contextUpdater.context.functions[getMethodName(it.value)]!! + }) + methodIdMap.clear() + } + + val conversionCodes = mutableMapOf, Code>() + val initCodes = mutableListOf() + val variablePlaceholder = "\$it" + val tempFieldPrefix = "___" + + for (it in contextUpdater.context.getVarsList().filter { !it.name.startsWith(tempFieldPrefix) }) { + val property = it.descriptor + property.isAccessible = true + val value = it.value ?: continue + val propertyType = property.returnType + val notNullType = propertyType.withNullability(false) + val functionInfo = handlers.asIterable().firstOrNull { it.key.matches(notNullType.toString()) }?.value + if (functionInfo != null) { + val codes = functionInfo.function.call(functionInfo.line, value, property)?.let { it as? List } + if (codes != null && codes.isNotEmpty()) { + var typeConverterCodes = TypeConverterCodes(codes) + if (typeConverterCodes.hasWildcard) { + val fullCode = typeConverterCodes.fullCode + var id = codeToMethodMap[fullCode] + if (id != null) { + typeConverterCodes = typeConverterCodes.withoutDeclarations() + } else { + id = codeToMethodMap.size + codeToMethodMap[fullCode] = id + } + if (typeConverterCodes.hasWildcard) + typeConverterCodes = typeConverterCodes.replaceWildcard("$id") + } + initCodes.addAll(typeConverterCodes.declarations) + var converterCode = typeConverterCodes.converter + if (propertyType.isMarkedNullable) + converterCode = converterCode.replace(variablePlaceholder, "$variablePlaceholder!!") + conversionCodes[property] = converterCode + continue + } + } + if (propertyType.isMarkedNullable) { + conversionCodes[property] = "$variablePlaceholder!!" + } + } + + if (!conversionCodes.isEmpty()) { + val initCode = initCodes.joinToLines() + val tempFieldsCode = conversionCodes + .map { "val $tempFieldPrefix${it.key.name} = ${it.key.name}" } + .joinToLines() + + val newFieldsCode = conversionCodes + .mapValues { it.value.replace(variablePlaceholder, "$tempFieldPrefix${it.key.name}") } + .map { + val valOrVar = if (it.key is KMutableProperty) "var" else "val" + "$valOrVar ${it.key.name} = ${it.value}" + } + .joinToLines() + + return listOf("$initCode\n$tempFieldsCode", newFieldsCode) + } + return emptyList() + } +} \ No newline at end of file diff --git a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt index ddefafc01..aa104942c 100644 --- a/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt +++ b/src/main/kotlin/org/jetbrains/kotlin/jupyter/util.kt @@ -1,11 +1,15 @@ package org.jetbrains.kotlin.jupyter -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.async -import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.* +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation import org.slf4j.Logger import java.io.File +import kotlin.script.experimental.api.ResultWithDiagnostics +import kotlin.script.experimental.api.ScriptDiagnostic +import kotlin.script.experimental.api.SourceCode +import kotlin.script.experimental.jvm.KJvmEvaluatedSnippet +import kotlin.script.experimental.util.LinkedSnippet +import kotlin.script.experimental.util.toList fun catchAll(body: () -> T): T? = try { body() @@ -37,7 +41,89 @@ fun Deferred.awaitBlocking(): T = if (isCompleted) getCompleted() else ru fun String.parseIniConfig() = split("\n").map { it.split('=') }.filter { it.count() == 2 }.map { it[0] to it[1] }.toMap() +fun List.joinToLines() = joinToString("\n") + fun File.tryReadIniConfig() = existsOrNull()?.let { catchAll { it.readText().parseIniConfig() } } + + + +fun LinkedSnippet?.instances() = this.toList { it.result.scriptInstance }.filterNotNull() + +fun generateDiagnostic(fromLine: Int, fromCol: Int, toLine: Int, toCol: Int, message: String, severity: String) = + ScriptDiagnostic( + ScriptDiagnostic.unspecifiedError, + message, + ScriptDiagnostic.Severity.valueOf(severity), + null, + SourceCode.Location(SourceCode.Position(fromLine, fromCol), SourceCode.Position(toLine, toCol)) + ) + +fun withPath(path: String?, diagnostics: List): List = + diagnostics.map { it.copy(sourcePath = path) } + +internal data class CompilationErrors( + val message: String, + val location: CompilerMessageLocation? +) + +internal fun ResultWithDiagnostics.getErrors(): CompilationErrors { + val filteredReports = reports.filter { + it.code != ScriptDiagnostic.incompleteCode + } + + return CompilationErrors( + filteredReports.joinToString("\n") { report -> + report.location?.let { loc -> + CompilerMessageLocation.create( + report.sourcePath, + loc.start.line, + loc.start.col, + loc.end?.line, + loc.end?.col, + null + )?.toStringExt()?.let { + "$it " + } + }.orEmpty() + report.message + }, + filteredReports.firstOrNull { + when (it.severity) { + ScriptDiagnostic.Severity.ERROR -> true + ScriptDiagnostic.Severity.FATAL -> true + else -> false + } + }?.let { + val loc = it.location ?: return@let null + CompilerMessageLocation.create( + it.sourcePath, + loc.start.line, + loc.start.col, + loc.end?.line, + loc.end?.col, + null + ) + } + ) +} + + +/** + * Converts its receiver to string with regard to its [CompilerMessageLocation.lineEnd] and + * [CompilerMessageLocation.columnEnd] fields + * @receiver CompilerMessageLocation + * @return String + */ +fun CompilerMessageLocation.toStringExt(): String { + val start = + if (line == -1 && column == -1) "" + else "$line:$column" + val end = + if (lineEnd == -1 && columnEnd == -1) "" + else if (lineEnd == line) " - $columnEnd" + else " - $lineEnd:$columnEnd" + val loc = if (start.isEmpty() && end.isEmpty()) "" else " ($start$end)" + return path + loc +} diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/executeTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/executeTests.kt index 46e0b4491..c6fdb63fa 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/executeTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/executeTests.kt @@ -1,7 +1,10 @@ package org.jetbrains.kotlin.jupyter.test import com.beust.klaxon.JsonObject -import org.jetbrains.kotlin.jupyter.* +import org.jetbrains.kotlin.jupyter.JupyterSockets +import org.jetbrains.kotlin.jupyter.Message +import org.jetbrains.kotlin.jupyter.get +import org.jetbrains.kotlin.jupyter.jsonObject import org.junit.Assert import org.junit.Test import org.zeromq.ZMQ @@ -98,4 +101,25 @@ class ExecuteTests : KernelServerTestsBase() { val res = doExecute(code, false, ::checker) Assert.assertNull(res) } + + @Test + fun testOutputStrings() { + val code = """ + for (i in 1..5) { + Thread.sleep(200) + println("text" + i) + } + """.trimIndent() + + fun checker(ioPub: ZMQ.Socket) { + for (i in 1..5) { + val msg = ioPub.receiveMessage() + Assert.assertEquals("stream", msg.type()) + Assert.assertEquals("text$i" + System.lineSeparator(), msg.content!!["text"]) + } + } + + val res = doExecute(code, false, ::checker) + Assert.assertNull(res) + } } \ No newline at end of file diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt index 6191339b2..b592bca29 100644 --- a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/replTests.kt @@ -3,78 +3,236 @@ package org.jetbrains.kotlin.jupyter.test import com.beust.klaxon.JsonObject import com.beust.klaxon.Parser import jupyter.kotlin.MimeTypedResult +import kotlinx.coroutines.runBlocking import org.jetbrains.kotlin.jupyter.* -import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResultSuccess +import org.jetbrains.kotlin.jupyter.repl.completion.CompletionResult +import jupyter.kotlin.receivers.ConstReceiver +import org.jetbrains.kotlin.jupyter.repl.completion.ListErrorsResult import org.junit.Assert import org.junit.Ignore import org.junit.Test -import kotlin.test.assertEquals -import kotlin.test.assertFails -import kotlin.test.assertNotNull +import java.io.File +import kotlin.script.experimental.api.SourceCode +import kotlin.test.* class ReplTest { - fun replWithResolver() = ReplForJupyter(classpath, ResolverConfig(defaultRepositories, + private fun replWithResolver() = ReplForJupyterImpl(classpath, ResolverConfig(defaultRepositories, parserLibraryDescriptors(readLibraries().toMap()).asDeferred())) + private fun assertEq(expected: T, actual: T, message: String? = null) = assertEquals(expected, actual, message) + @Test - fun TestRepl() { - val repl = ReplForJupyter(classpath) + fun testRepl() { + val repl = ReplForJupyterImpl(classpath) repl.eval("val x = 3") val res = repl.eval("x*2") assertEquals(6, res.resultValue) } @Test - fun TestDependsOnAnnotation() { - val repl = ReplForJupyter(classpath) + fun testPropertiesGeneration() { + val repl = ReplForJupyterImpl(classpath) + // Note, this test should actually fail with ReplEvalRuntimeException, but 'cause of eval/compile + // histories are out of sync, it fails with another exception. This test shows the wrong behavior and + // should be fixed after fixing https://youtrack.jetbrains.com/issue/KT-36397 + + // In fact, this shouldn't compile, but because of bug in compiler it fails in runtime + assertFailsWith(ReplEvalRuntimeException::class) { + repl.eval(""" + fun stack(vararg tup: Int): Int = tup.sum() + val X = 1 + val x = stack(1, X) + """.trimIndent()) + + print("") + } + } + + @Test + fun testError() { + val repl = ReplForJupyterImpl(classpath) + try { + repl.eval(""" + val foobar = 78 + val foobaz = "dsdsda" + val ddd = ppp + val ooo = foobar + """.trimIndent()) + } catch (ex: ReplCompilerException) { + val diag = ex.firstDiagnostics + val location = diag?.location ?: fail("Location should not be null") + val message = ex.message + + val expectedLocation = SourceCode.Location(SourceCode.Position(3, 11), SourceCode.Position(3, 14)) + val expectedMessage = "Line_1.jupyter.kts (3:11 - 14) Unresolved reference: ppp" + + assertEquals(expectedLocation, location) + assertEquals(expectedMessage, message) + + return + } + + fail("Test should fail with ReplCompilerException") + } + + @Test + fun testReplWithReceiver() { + val value = 5 + val cp = classpath + File(ConstReceiver::class.java.protectionDomain.codeSource.location.toURI().path) + val repl = ReplForJupyterImpl(cp, null, ConstReceiver(value)) + val res = repl.eval("value") + assertEquals(value, res.resultValue) + } + + @Test + fun testDependsOnAnnotation() { + val repl = ReplForJupyterImpl(classpath) repl.eval("@file:DependsOn(\"de.erichseifert.gral:gral-core:0.11\")") } @Test - fun TestScriptIsolation() { - val repl = ReplForJupyter(classpath) + fun testDependsOnAnnotationCompletion() { + val repl = ReplForJupyterImpl(classpath) + repl.eval(""" + @file:Repository("https://repo1.maven.org/maven2/") + @file:DependsOn("com.github.doyaaaaaken:kotlin-csv-jvm:0.7.3") + """.trimIndent()) + + val res = runBlocking { + var res2: CompletionResult? = null + repl.complete("import com.github.", 18) { res2 = it } + res2 + } + when(res) { + is CompletionResult.Success -> res.sortedMatches().contains("doyaaaaaken") + else -> fail("Completion should be successful") + } + } + + @Test + fun testScriptIsolation() { + val repl = ReplForJupyterImpl(classpath) assertFails { repl.eval("org.jetbrains.kotlin.jupyter.ReplLineMagics.use") } } @Test - fun TestDependsOnAnnotations() { - val repl = ReplForJupyter(classpath) + fun testDependsOnAnnotations() { + val repl = ReplForJupyterImpl(classpath) val sb = StringBuilder() - sb.appendln("@file:DependsOn(\"de.erichseifert.gral:gral-core:0.11\")") - sb.appendln("@file:Repository(\"https://repo.spring.io/libs-release\")") - sb.appendln("@file:DependsOn(\"org.jetbrains.kotlinx:kotlinx.html.jvm:0.5.12\")") + sb.appendLine("@file:DependsOn(\"de.erichseifert.gral:gral-core:0.11\")") + sb.appendLine("@file:Repository(\"https://repo.spring.io/libs-release\")") + sb.appendLine("@file:DependsOn(\"org.jetbrains.kotlinx:kotlinx.html.jvm:0.5.12\")") repl.eval(sb.toString()) } @Test - fun TestCompletion() { - val repl = ReplForJupyter(classpath) + fun testCompletionSimple() { + val repl = ReplForJupyterImpl(classpath) repl.eval("val foobar = 42") repl.eval("var foobaz = 43") - val result = repl.complete("val t = foo", 11) - if (result is CompletionResultSuccess) { - Assert.assertEquals(arrayListOf("foobar", "foobaz"), result.matches.sorted()) - } else { - Assert.fail("Result should be success") + runBlocking { repl.complete("val t = foo", 11) { + result -> + if (result is CompletionResult.Success) { + Assert.assertEquals(arrayListOf("foobar", "foobaz"), result.sortedMatches()) + } else { + Assert.fail("Result should be success") + } + } + } + } + + @Test + fun testNoCompletionAfterNumbers() { + val repl = ReplForJupyterImpl(classpath) + + runBlocking { repl.complete("val t = 42", 10) { + result -> + if (result is CompletionResult.Success) { + assertEq(emptyList(), result.sortedMatches()) + } else { + Assert.fail("Result should be success") + } + } + } + } + + @Test + fun testCompletionForImplicitReceivers() { + val repl = ReplForJupyterImpl(classpath) + repl.eval(""" + class AClass(val c_prop_x: Int) { + fun filter(xxx: (AClass).() -> Boolean): AClass { + return this + } + } + val AClass.c_prop_y: Int + get() = c_prop_x * c_prop_x + + fun AClass.c_meth_z(v: Int) = v * c_prop_y + val df = AClass(10) + val c_zzz = "some string" + """.trimIndent()) + runBlocking { + repl.complete("df.filter { c_ }", 14) { result -> + if (result is CompletionResult.Success) { + Assert.assertEquals(arrayListOf("c_meth_z(", "c_prop_x", "c_prop_y", "c_zzz"), result.sortedMatches()) + } else { + Assert.fail("Result should be success") + } + } } } @Test - fun TestOut() { - val repl = ReplForJupyter(classpath) - repl.eval("1+1", 1) + fun testErrorsList() { + val repl = ReplForJupyterImpl(classpath) + repl.eval(""" + data class AClass(val memx: Int, val memy: String) + data class BClass(val memz: String, val mema: AClass) + val foobar = 42 + var foobaz = "string" + val v = BClass("KKK", AClass(5, "25")) + """.trimIndent()) + runBlocking { + repl.listErrors(""" + val a = AClass("42", 3.14) + val b: Int = "str" + val c = foob + """.trimIndent()) {result -> + val actualErrors = result.errors.toList() + val path = actualErrors.first().sourcePath + Assert.assertEquals(withPath(path, listOf( + generateDiagnostic(1, 16, 1, 20, "Type mismatch: inferred type is String but Int was expected", "ERROR"), + generateDiagnostic(1, 22, 1, 26, "The floating-point literal does not conform to the expected type String", "ERROR"), + generateDiagnostic(2, 14, 2, 19, "Type mismatch: inferred type is String but Int was expected", "ERROR"), + generateDiagnostic(3, 9, 3, 13, "Unresolved reference: foob", "ERROR") + )), actualErrors) + } + } + } + + @Test + fun testEmptyErrorsListJson() { + val res = ListErrorsResult("someCode") + assertEquals("""{"errors":[],"code":"someCode"}""", res.toJson().toJsonString()) + } + + @Test + fun testOut() { + val repl = ReplForJupyterImpl(classpath) + repl.eval("1+1", null, 1) val res = repl.eval("Out[1]") assertEquals(2, res.resultValue) assertFails { repl.eval("Out[3]") } } @Test - fun TestOutputMagic() { - val repl = ReplForJupyter(classpath) + fun testOutputMagic() { + val repl = ReplForJupyterImpl(classpath) repl.preprocessCode("%output --max-cell-size=100500 --no-stdout") assertEquals(OutputConfig( cellOutputMaxSize = 100500, @@ -95,7 +253,7 @@ class ReplTest { } @Test - fun TestUseMagic() { + fun testUseMagic() { val lib1 = "mylib" to """ { "properties": { @@ -120,53 +278,87 @@ class ReplTest { "a": "temp", "b": "test" }, + "repositories": [ + "repo-${'$'}a" + ], "dependencies": [ - "path-${'$'}a", "path-${'$'}b" ], "imports": [ "otherPackage" + ], + "init": [ + "otherInit" ] } """.trimIndent() + val lib3 = "another" to """ + { + "properties": { + "v": "1" + }, + "dependencies": [ + "anotherDep" + ], + "imports": [ + "anotherPackage${'$'}v" + ], + "init": [ + "%use other(b=release, a=debug)", + "anotherInit" + ] + } + """.trimIndent() val parser = Parser.default() - val libJsons = arrayOf(lib1, lib2).map { it.first to parser.parse(StringBuilder(it.second)) as JsonObject }.toMap() + val libJsons = arrayOf(lib1, lib2, lib3).map { it.first to parser.parse(StringBuilder(it.second)) as JsonObject }.toMap() - val repl = ReplForJupyter(classpath, ResolverConfig(defaultRepositories, parserLibraryDescriptors(libJsons).asDeferred())) - val res = repl.preprocessCode("%use mylib(1.0), other(b=release, a=debug)").trimIndent() - val libs = repl.librariesCodeGenerator.getProcessedLibraries() - assertEquals("", res) - assertEquals(2, libs.count()) - arrayOf( + val repl = ReplForJupyterImpl(classpath, ResolverConfig(defaultRepositories, parserLibraryDescriptors(libJsons).asDeferred())) + val res = repl.preprocessCode("%use mylib(1.0), another") + assertEquals("", res.code) + val inits = arrayOf( """ @file:DependsOn("artifact1:1.0") @file:DependsOn("artifact2:1.0") import package1 import package2 - code1 - code2 - """, + """, + "code1", + "code2", + """ + @file:DependsOn("anotherDep") + import anotherPackage1 + """, """ - @file:DependsOn("path-debug") + @file:Repository("repo-debug") @file:DependsOn("path-release") import otherPackage - """ - ).forEachIndexed { index, expected -> - Assert.assertEquals(expected.trimIndent(), libs[index].code.trimEnd().convertCRLFtoLF()) + """, + "otherInit", + "anotherInit" + ) + assertEquals(inits.count(), res.initCodes.count()) + inits.forEachIndexed { index, expected -> + Assert.assertEquals(expected.trimIndent(), res.initCodes[index].trimEnd().convertCRLFtoLF()) } } @Test - fun TestLetsPlot() { + fun testLetsPlot() { val repl = replWithResolver() val code1 = "%use lets-plot" val code2 = """lets_plot(mapOf("cat" to listOf("a", "b")))""" - val res1 = repl.eval(code1) - Assert.assertEquals(1, res1.displayValues.count()) + val displays = mutableListOf() + fun displayHandler(display: Any) { + displays.add(display) + } + + val res1 = repl.eval(code1, ::displayHandler) + Assert.assertEquals(1, displays.count()) + displays.clear() Assert.assertNull(res1.resultValue) - val res2 = repl.eval(code2) - Assert.assertEquals(0, res2.displayValues.count()) + val res2 = repl.eval(code2, ::displayHandler) + Assert.assertEquals(0, displays.count()) val mime = res2.resultValue as? MimeTypedResult assertNotNull(mime) assertEquals(1, mime.size) @@ -175,17 +367,21 @@ class ReplTest { } @Test - fun TestTwoLibrariesInUse() { + fun testTwoLibrariesInUse() { val repl = replWithResolver() val code = "%use lets-plot, krangl" - val res = repl.eval(code) - assertEquals(1, res.displayValues.count()) + val displays = mutableListOf() + fun displayHandler(display: Any) { + displays.add(display) + } + repl.eval(code, ::displayHandler) + assertEquals(1, displays.count()) } @Ignore @Test //TODO: https://github.com/Kotlin/kotlin-jupyter/issues/25 - fun TestKranglImportInfixFun() { + fun testKranglImportInfixFun() { val repl = replWithResolver() val code = """%use krangl "a" to {it["a"]}""" @@ -193,6 +389,16 @@ class ReplTest { assertNotNull(res.resultValue) } + @Test + fun testNullableErasure() { + val repl = replWithResolver() + val code1 = "val a: Int? = 3" + repl.eval(code1) + val code2 = "a+2" + val res = repl.eval(code2).resultValue + assertEquals(5, res) + } + private fun String.convertCRLFtoLF(): String { return replace("\r\n", "\n") } diff --git a/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt new file mode 100644 index 000000000..13e1be703 --- /dev/null +++ b/src/test/kotlin/org/jetbrains/kotlin/jupyter/test/typeProviderTests.kt @@ -0,0 +1,57 @@ +package org.jetbrains.kotlin.jupyter.test + +import com.beust.klaxon.JsonObject +import com.beust.klaxon.Parser +import jupyter.kotlin.receivers.TypeProviderReceiver +import org.jetbrains.kotlin.jupyter.* +import org.junit.Test +import java.io.File +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + + +class TypeConverterTests { + + @Test + fun test() { + + val parser = Parser.default() + val descriptor = """ + { + "typeConverters": { + "kotlin.collections.List": "generateCode(${'$'}it)" + } + } + """.trimIndent() + val cp = classpath + File(TypeProviderReceiver::class.java.protectionDomain.codeSource.location.toURI().path) + val libJsons = mapOf("mylib" to parser.parse(StringBuilder(descriptor)) as JsonObject) + val repl = ReplForJupyterImpl(cp, ResolverConfig(defaultRepositories, parserLibraryDescriptors(libJsons).asDeferred()), TypeProviderReceiver()) + + // create list 'l' of size 3 + val code1 = """ + %use mylib + val l = listOf(1,2,3) + """.trimIndent() + repl.eval(code1) + assertEquals(3, repl.eval("l.value2").resultValue) + + // create list 'q' of the same size 3 + repl.eval("val q = l.asReversed()") + assertEquals(1, repl.eval("q.value2").resultValue) + + // check that 'l' and 'q' have the same types + assertEquals(3, repl.eval("""var a = l + a = q + a.value0 + """.trimMargin()).resultValue) + + // create a list of size 6 + repl.eval("val w = l + a") + assertEquals(3, repl.eval("w.value3").resultValue) + + // check that 'value3' is not available for list 'l' + assertFailsWith(ReplCompilerException::class) { + repl.eval("l.value3") + } + } +} \ No newline at end of file