OpenAPI -> experimental

README.md

@@ -219,21 +219,21 @@ Check it out on [**Datalore**](https://datalore.jetbrains.com/view/notebook/vq5j
 
 Explore [**more examples here**](examples).
 
-## Kotlin, Kotlin Jupyter, OpenAPI, Arrow and JDK versions
+## Kotlin, Kotlin Jupyter, Arrow, and JDK versions
 
 This table shows the mapping between main library component versions and minimum supported Java versions.
 
-| Kotlin DataFrame Version | Minimum Java Version | Kotlin Version | Kotlin Jupyter Version | OpenAPI version | Apache Arrow version |
-|--------------------------|----------------------|----------------|------------------------|-----------------|----------------------|
-| 0.10.0                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.10.1                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.11.0                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.11.1                   | 8                    | 1.8.20         | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.12.0                   | 8                    | 1.9.0          | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.12.1                   | 8                    | 1.9.0          | 0.11.0-358             | 3.0.0           | 11.0.0               |
-| 0.13.1                   | 8                    | 1.9.22         | 0.12.0-139             | 3.0.0           | 15.0.0               |
-| 0.14.1                   | 8                    | 2.0.20         | 0.12.0-139             | 3.0.0           | 17.0.0               |
-| 0.15.0                   | 8                    | 2.0.20         | 0.12.0-139             | 3.0.0           | 18.1.0               |
+| Kotlin DataFrame Version | Minimum Java Version | Kotlin Version | Kotlin Jupyter Version | Apache Arrow version |
+|--------------------------|----------------------|----------------|------------------------|----------------------|
+| 0.10.0                   | 8                    | 1.8.20         | 0.11.0-358             | 11.0.0               |
+| 0.10.1                   | 8                    | 1.8.20         | 0.11.0-358             | 11.0.0               |
+| 0.11.0                   | 8                    | 1.8.20         | 0.11.0-358             | 11.0.0               |
+| 0.11.1                   | 8                    | 1.8.20         | 0.11.0-358             | 11.0.0               |
+| 0.12.0                   | 8                    | 1.9.0          | 0.11.0-358             | 11.0.0               |
+| 0.12.1                   | 8                    | 1.9.0          | 0.11.0-358             | 11.0.0               |
+| 0.13.1                   | 8                    | 1.9.22         | 0.12.0-139             | 15.0.0               |
+| 0.14.1                   | 8                    | 2.0.20         | 0.12.0-139             | 17.0.0               |
+| 0.15.0                   | 8                    | 2.0.20         | 0.12.0-139             | 18.1.0               |
 
 ## Code of Conduct
 

build.gradle.kts

@@ -52,10 +52,12 @@ dependencies {
     api(projects.core)
     api(projects.dataframeArrow)
     api(projects.dataframeExcel)
-    api(projects.dataframeOpenapi)
     api(projects.dataframeJdbc)
     api(projects.dataframeCsv)
 
+    // experimental, so not included by default:
+    // api(projects.dataframeOpenapi)
+
     kover(projects.core)
     kover(projects.dataframeArrow)
     kover(projects.dataframeExcel)

core/api/core.api

@@ -240,6 +240,7 @@ public abstract interface annotation class org/jetbrains/kotlinx/dataframe/annot
 
 public abstract interface annotation class org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema : java/lang/annotation/Annotation {
 	public abstract fun csvOptions ()Lorg/jetbrains/kotlinx/dataframe/annotations/CsvOptions;
+	public abstract fun enableExperimentalOpenApi ()Z
 	public abstract fun jdbcOptions ()Lorg/jetbrains/kotlinx/dataframe/annotations/JdbcOptions;
 	public abstract fun jsonOptions ()Lorg/jetbrains/kotlinx/dataframe/annotations/JsonOptions;
 	public abstract fun name ()Ljava/lang/String;
@@ -5941,7 +5942,10 @@ public final class org/jetbrains/kotlinx/dataframe/jupyter/ImportDataSchemaKt {
 
 public final class org/jetbrains/kotlinx/dataframe/jupyter/JupyterConfiguration {
 	public fun <init> ()V
+	public fun <init> (Z)V
+	public synthetic fun <init> (ZILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public final fun getDisplay ()Lorg/jetbrains/kotlinx/dataframe/io/DisplayConfiguration;
+	public final fun getEnableExperimentalOpenApi ()Z
 	public final fun getVersion ()Ljava/lang/String;
 	public final fun invoke (Lkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/jupyter/JupyterConfiguration;
 }

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema.kt

@@ -27,6 +27,7 @@ import org.jetbrains.kotlinx.dataframe.io.JSON
  * @param csvOptions options to parse CSV data. Not used when data is not Csv
  * @param jsonOptions options to parse JSON data. Not used when data is not Json
  * @param jdbcOptions options to parse data from a database via JDBC. Not used when data is not stored in the database
+ * @param enableExperimentalOpenApi Can be set to `true` to enable experimental OpenAPI 3.0.0 types support
  */
 @Retention(AnnotationRetention.SOURCE)
 @Target(AnnotationTarget.FILE)
@@ -40,6 +41,7 @@ public annotation class ImportDataSchema(
     val csvOptions: CsvOptions = CsvOptions(','),
     val jsonOptions: JsonOptions = JsonOptions(),
     val jdbcOptions: JdbcOptions = JdbcOptions(),
+    val enableExperimentalOpenApi: Boolean = false,
 )
 
 public enum class DataSchemaVisibility {

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterConfiguration.kt

@@ -3,7 +3,10 @@ package org.jetbrains.kotlinx.dataframe.jupyter
 import org.jetbrains.kotlinx.dataframe.BuildConfig
 import org.jetbrains.kotlinx.dataframe.io.DisplayConfiguration
 
-public class JupyterConfiguration {
+public class JupyterConfiguration(
+    /** If true, experimental OpenAPI 3.0.0 types support via importDataSchema() is enabled. Can be set via `%use dataframe(..., enableExperimentalOpenApi=true)` */
+    public val enableExperimentalOpenApi: Boolean = false,
+) {
     public val display: DisplayConfiguration = DisplayConfiguration()
 
     /** Version of the library. */

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/importDataSchema.kt

@@ -20,9 +20,14 @@ internal val importDataSchema =
     """
     /** Import the type-only data schema from [url]. */
     fun importDataSchema(url: URL, name: String) {
-        val formats = listOf(
-            OpenApi(),
+        val formats = listOfNotNull(
+            if (dataFrameConfig.enableExperimentalOpenApi) OpenApi() else null,
         )
+
+        require(formats.isNotEmpty()) { 
+            "importDataSchema() did not find any supported type-only data schema generation providers (`SupportedCodeGenerationFormat`). If you were looking for OpenAPI 3.0.0 types, set `%use dataframe(..., enableExperimentalOpenApi=true)`." 
+        }   
+
         val codeGenResult = org.jetbrains.dataframe.impl.codeGen.CodeGenerator.urlCodeGenReader(url, formats)
         when (codeGenResult) {
             is org.jetbrains.kotlinx.dataframe.impl.codeGen.CodeGenerationReadResult.Success -> {

dataframe-geo/README.md

@@ -3,6 +3,6 @@
 This module, published as `dataframe-geo`, contains all logic and tests for DataFrame to be able to work 
 with geographical data.
 
-Experimental.
+**Experimental**.
 
 This module targets java 11 because of the restriction from `org.jetbrains.kotlin.jupyter`.

dataframe-jupyter/build.gradle.kts

@@ -1,4 +1,3 @@
-import org.jetbrains.kotlin.gradle.tasks.BaseKotlinCompile
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
@@ -27,6 +26,10 @@ dependencies {
     testImplementation(projects.dataframeCsv)
     testImplementation(projects.dataframeExcel)
     testImplementation(projects.dataframeJdbc)
+    // experimental
+    testImplementation(projects.dataframeOpenapiGenerator)
+    testImplementation(projects.dataframeOpenapi)
+
     testImplementation(libs.kotestAssertions) {
         exclude("org.jetbrains.kotlin", "kotlin-stdlib-jdk8")
     }

dataframe-jupyter/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt

@@ -67,8 +67,10 @@ internal class Integration(private val notebook: Notebook, private val options:
     val version = options["v"]
 
     // TODO temporary settings while these experimental modules are being developed
+
     private val enableExperimentalCsv = options["enableExperimentalCsv"]
     private val enableExperimentalGeo = options["enableExperimentalGeo"]
+    private val enableExperimentalOpenApi = options["enableExperimentalOpenApi"]
 
     private fun KotlinKernelHost.updateImportDataSchemaVariable(
         importDataSchema: ImportDataSchema,
@@ -161,14 +163,19 @@ internal class Integration(private val notebook: Notebook, private val options:
                 repositories("https://repo.osgeo.org/repository/release")
                 dependencies("org.jetbrains.kotlinx:dataframe-geo:$version")
             }
+            if (enableExperimentalOpenApi?.toBoolean() == true) {
+                println("Enabling experimental OpenAPI 3.0.0 module: dataframe-openapi")
+                dependencies(
+                    "org.jetbrains.kotlinx:dataframe-openapi:$version",
+                    "org.jetbrains.kotlinx:dataframe-openapi-generator:$version",
+                )
+            }
             dependencies(
                 "org.jetbrains.kotlinx:dataframe-core:$version",
                 "org.jetbrains.kotlinx:dataframe-csv:$version",
                 "org.jetbrains.kotlinx:dataframe-excel:$version",
                 "org.jetbrains.kotlinx:dataframe-jdbc:$version",
                 "org.jetbrains.kotlinx:dataframe-arrow:$version",
-                "org.jetbrains.kotlinx:dataframe-openapi:$version",
-                "org.jetbrains.kotlinx:dataframe-openapi-generator:$version",
             )
         }
 
@@ -181,7 +188,7 @@ internal class Integration(private val notebook: Notebook, private val options:
             )
         }
         val codeGen = ReplCodeGenerator.create()
-        val config = JupyterConfiguration()
+        val config = JupyterConfiguration(enableExperimentalOpenApi = enableExperimentalOpenApi?.toBoolean() == true)
 
         if (notebook.jupyterClientType == JupyterClientType.KOTLIN_NOTEBOOK) {
             config.display.isolatedOutputs = true

dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/SampleNotebooksTests.kt

@@ -100,6 +100,22 @@ class SampleNotebooksTests : DataFrameJupyterTest() {
             ),
         )
 
+    @Test
+    fun json() =
+        exampleTest(
+            dir = "json",
+            notebookName = "KeyValueAndOpenApi",
+            cellClause = CellClause {
+                // skip OOM cells
+                it.metadata.tags?.contains("skiptest") != true
+            },
+            replacer = CodeReplacer.byMap(
+                testFile("json", "api_guru_list.json"),
+                testFile("json", "apiGuruMetrics.json"),
+                testFile("json", "ApiGuruOpenApi.yaml"),
+            ),
+        )
+
     private fun doTest(
         notebookPath: String,
         replacer: CodeReplacer,

dataframe-openapi-generator/README.md

@@ -1,7 +1,7 @@
 ## :dataframe-openapi-generator
 
-This module, published as `dataframe-openapi-generator` contains all logic and tests for DataFrame to be able to import
-OpenAPI specifications as auto-generated data schemas. This module is a sister module to
+This **experimental** module, published as `dataframe-openapi-generator` contains all logic and tests for DataFrame to be able to import
+OpenAPI 3.0.0 specifications as auto-generated data schemas. This module is a sister module to
 [`dataframe-openapi`](../dataframe-openapi):
 
 - `dataframe-openapi-generator` is used as a dependency of the Gradle plugin and Jupyter plugin to be able to generate

dataframe-openapi/README.md

@@ -1,7 +1,7 @@
 ## :dataframe-openapi
 
-This module, published as `dataframe-openapi` contains some functions to be able to use auto-generated
-data schemas from OpenAPI specifications. This module is a sister module to
+This **experimental** module, published as `dataframe-openapi` contains some functions to be able to use auto-generated
+data schemas from OpenAPI 3.0.0 specifications. This module is a sister module to
 [`dataframe-openapi-generator`](../dataframe-openapi-generator):
 
 - `dataframe-openapi-generator` is used as a dependency of the Gradle plugin and Jupyter plugin to be able to generate

docs/StardustDocs/topics/gettingStartedGradleAdvanced.md

@@ -128,8 +128,11 @@ dependencies {
     implementation("org.jetbrains.kotlinx:dataframe-excel:%dataFrameVersion%")
     implementation("org.jetbrains.kotlinx:dataframe-jdbc:%dataFrameVersion%")
     implementation("org.jetbrains.kotlinx:dataframe-arrow:%dataFrameVersion%")
+
+    // experimental
     implementation("org.jetbrains.kotlinx:dataframe-openapi:%dataFrameVersion%")
 
+    // experimental
     // This artifact is only needed to directly call functions that generate @DataSchema code from OpenAPI specifications
     // It's used by Gradle and KSP plugins internally.
     // Your project needs dataframe-openapi to use generated code
@@ -149,7 +152,14 @@ dependencies {
     implementation 'org.jetbrains.kotlinx:dataframe-excel:%dataFrameVersion%'
     implementation 'org.jetbrains.kotlinx:dataframe-jdbc:%dataFrameVersion%'
     implementation 'org.jetbrains.kotlinx:dataframe-arrow:%dataFrameVersion%'
+
+    // experimental
     implementation 'org.jetbrains.kotlinx:dataframe-openapi:%dataFrameVersion%'
+
+    // experimental
+    // This artifact is only needed to directly call functions that generate @DataSchema code from OpenAPI specifications
+    // It's used by Gradle and KSP plugins internally.
+    // Your project needs dataframe-openapi to use generated code
     implementation 'org.jetbrains.kotlinx:dataframe-openapi-generator:%dataFrameVersion%'
 }
 ```

docs/StardustDocs/topics/schemas.md

@@ -38,9 +38,9 @@ Here's a list of the most popular use cases with Data Schemas.
 * [**Schema Definitions from SQL Databases in Gradle Project**](schemasImportSqlGradle.md) <br/>
   When you need to take data from the SQL database.
 
-* [**Import OpenAPI Schemas in Gradle Project**](schemasImportOpenApiGradle.md) <br/>
+* [**Import OpenAPI 3.0.0 Schemas (Experimental) in Gradle Project**](schemasImportOpenApiGradle.md) <br/>
   When you need to take data from the endpoint with OpenAPI Schema.
 
-* [**Import Data Schemas, e.g. from OpenAPI, in Jupyter**](schemasImportOpenApiJupyter.md) <br/>
+* [**Import Data Schemas, e.g. from OpenAPI 3.0.0 (Experimental), in Jupyter**](schemasImportOpenApiJupyter.md) <br/>
   Similar to [importing OpenAPI Data Schemas in Gradle projects](schemasImportOpenApiGradle.md), 
   you can also do this in Jupyter Notebooks.

docs/StardustDocs/topics/schemasImportOpenApiGradle.md

@@ -1,7 +1,11 @@
-[//]: # (title: Import OpenAPI Schemas in Gradle project)
+[//]: # (title: Import OpenAPI Schemas in Gradle project (Experimental))
 
 <!---IMPORT org.jetbrains.kotlinx.dataframe.samples.api.Schemas-->
 
+<warning>
+OpenAPI 3.0.0 schema support is marked as experimental. It might change or be removed in the future.
+</warning>
+
 JSON schema inference is great, but it's not perfect. However, more and more APIs offer
 [OpenAPI (Swagger)](https://swagger.io/) specifications. 
 
@@ -27,6 +31,7 @@ OpenAPI type schemas can be generated using both methods described above:
 @file:ImportDataSchema(
     path = "https://petstore3.swagger.io/api/v3/openapi.json",
     name = "PetStore",
+    enableExperimentalOpenApi = true,
 )
 
 import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema
@@ -38,6 +43,7 @@ dataframes {
         data = "https://petstore3.swagger.io/api/v3/openapi.json"
         name = "PetStore"
     }
+    enableExperimentalOpenApi = true
 }
 ```
 

docs/StardustDocs/topics/schemasImportOpenApiJupyter.md

@@ -2,8 +2,16 @@
 
 <!---IMPORT org.jetbrains.kotlinx.dataframe.samples.api.Schemas-->
 
+<warning>
+OpenAPI 3.0.0 schema support is marked as experimental. It might change or be removed in the future.
+</warning>
+
 Similar to [importing OpenAPI Data Schemas in Gradle projects](schemasImportOpenApiGradle.md),
-you can also do this in Jupyter Notebooks. 
+you can also do this in Jupyter Notebooks.
+This requires enabling the `enableExperimentalOpenApi` setting, like:
+```
+%use dataframe(..., enableExperimentalOpenApi=true)
+```
 
 There is only a slight difference in notation:
 
@@ -13,7 +21,7 @@ Import the schema using any path (`String`), `URL`, or `File`:
 val PetStore = importDataSchema("https://petstore3.swagger.io/api/v3/openapi.json")
 ```
 
-and then from next cell you run and onwards, you can call, for example:
+and then from the next cell you run and onwards, you can call, for example:
 
 ```kotlin
 val df = PetStore.Pet.readJson("https://petstore3.swagger.io/api/v3/pet/findByStatus?status=available")

examples/idea-examples/json/build.gradle.kts

@@ -20,6 +20,9 @@ repositories {
 dependencies {
     // implementation("org.jetbrains.kotlinx:dataframe:X.Y.Z")
     implementation(project(":"))
+
+    // explicitly depend on openApi
+    implementation(projects.dataframeOpenapi)
 }
 
 tasks.withType<KotlinCompile> {
@@ -53,4 +56,6 @@ dataframes {
         // name is still needed to get the full path
         name = "org.jetbrains.kotlinx.dataframe.examples.openapi.ApiGuruOpenApiGradle"
     }
+
+    enableExperimentalOpenApi = true
 }

examples/idea-examples/json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/openapi/keyValueApiGuru.kt

@@ -2,15 +2,17 @@
     // Using just a sample since the full file will cause OOM errors
     path = "src/main/resources/ApiGuruSample.json",
     name = "APIsNoKeyValue",
+    enableExperimentalOpenApi = true,
 )
 @file:ImportDataSchema(
     // Now we can use the full file; either a URL or a local path
     path = "src/main/resources/api_guru_list.json",
     name = "APIsKeyValue",
     jsonOptions = JsonOptions(
         // paths in the json that should be converted to KeyValue columns
-        keyValuePaths = ["""$""", """$[*]["versions"]"""]
-    )
+        keyValuePaths = ["""$""", """$[*]["versions"]"""],
+    ),
+    enableExperimentalOpenApi = true,
 )
 
 package org.jetbrains.kotlinx.dataframe.examples.openapi

examples/idea-examples/json/src/main/kotlin/org/jetbrains/kotlinx/dataframe/examples/openapi/openApiApiGuru.kt

@@ -1,10 +1,12 @@
 @file:ImportDataSchema(
     path = "src/main/resources/ApiGuruOpenApi.yaml",
     name = "ApiGuruOpenApiKsp",
+    enableExperimentalOpenApi = true,
 )
 @file:ImportDataSchema(
     path = "https://raw.githubusercontent.com/1Password/connect/aac5e44b27570036e6b56e9f5b2a398a824ae5fc/docs/openapi/spec.yaml",
     name = "OnePassword",
+    enableExperimentalOpenApi = true,
 )
 
 package org.jetbrains.kotlinx.dataframe.examples.openapi