Sync Spark Connect pb from upstream (pre Spark 4.0.0)

kyuubi-relocated-spark-connect-rpc/src/main/protobuf/README.md

@@ -1,5 +1,5 @@
 proto files are copied from
-https://github.com/apache/spark/tree/5b2d2149b615acdd8730547a1f24c2b637222545/sql/connect/common/src/main/protobuf
+https://github.com/apache/spark/tree/8a1f4acead0a580142152656913829700b710652/sql/connect/common/src/main/protobuf
 
 and with one additional change in each proto file
 ```patch

kyuubi-relocated-spark-connect-rpc/src/main/protobuf/spark/connect/base.proto

@@ -25,6 +25,7 @@ import "spark/connect/common.proto";
 import "spark/connect/expressions.proto";
 import "spark/connect/relations.proto";
 import "spark/connect/types.proto";
+import "spark/connect/ml.proto";
 
 option java_multiple_files = true;
 option java_package = "org.apache.kyuubi.shaded.spark.connect.proto";
@@ -94,6 +95,7 @@ message AnalyzePlanRequest {
     Persist persist = 14;
     Unpersist unpersist = 15;
     GetStorageLevel get_storage_level = 16;
+    JsonToDDL json_to_ddl = 18;
   }
 
   message Schema {
@@ -199,6 +201,11 @@ message AnalyzePlanRequest {
     // (Required) The logical plan to get the storage level.
     Relation relation = 1;
   }
+
+  message JsonToDDL {
+    // (Required) The JSON formatted string to be converted to DDL.
+    string json_string = 1;
+  }
 }
 
 // Response to performing analysis of the query. Contains relevant metadata to be able to
@@ -224,6 +231,7 @@ message AnalyzePlanResponse {
     Persist persist = 12;
     Unpersist unpersist = 13;
     GetStorageLevel get_storage_level = 14;
+    JsonToDDL json_to_ddl = 16;
   }
 
   message Schema {
@@ -275,6 +283,10 @@ message AnalyzePlanResponse {
     // (Required) The StorageLevel as a result of get_storage_level request.
     StorageLevel storage_level = 1;
   }
+
+  message JsonToDDL {
+    string ddl_string = 1;
+  }
 }
 
 // A request to be executed by the service.
@@ -384,6 +396,9 @@ message ExecutePlanResponse {
     // Response for command that checkpoints a DataFrame.
     CheckpointCommandResult checkpoint_command_result = 19;
 
+    // ML command response
+    MlCommandResult ml_command_result = 20;
+
     // Support arbitrary result objects.
     google.protobuf.Any extension = 999;
   }
@@ -514,6 +529,9 @@ message ConfigRequest {
   message Set {
     // (Required) The config key-value pairs to set.
     repeated KeyValue pairs = 1;
+
+    // (Optional) Whether to ignore failures.
+    optional bool silent = 2;
   }
 
   message Get {
@@ -522,7 +540,7 @@ message ConfigRequest {
   }
 
   message GetWithDefault {
-    // (Required) The config key-value paris to get. The value will be used as the default value.
+    // (Required) The config key-value pairs to get. The value will be used as the default value.
     repeated KeyValue pairs = 1;
   }
 
@@ -913,6 +931,20 @@ message ReleaseSessionRequest {
   // can be used for language or version specific information and is only intended for
   // logging purposes and will not be interpreted by the server.
   optional string client_type = 3;
+
+  // Signals the server to allow the client to reconnect to the session after it is released.
+  //
+  // By default, the server tombstones the session upon release, preventing reconnections and
+  // fully cleaning the session state.
+  //
+  // If this flag is set to true, the server may permit the client to reconnect to the session
+  // post-release, even if the session state has been cleaned. This can result in missing state,
+  // such as Temporary Views, Temporary UDFs, or the Current Catalog, in the reconnected session.
+  //
+  // Use this option sparingly and only when the client fully understands the implications of
+  // reconnecting to a released session. The client must ensure that any queries executed do not
+  // rely on the session state prior to its release.
+  bool allow_reconnect = 4;
 }
 
 // Next ID: 3

kyuubi-relocated-spark-connect-rpc/src/main/protobuf/spark/connect/commands.proto

@@ -21,6 +21,7 @@ import "google/protobuf/any.proto";
 import "spark/connect/common.proto";
 import "spark/connect/expressions.proto";
 import "spark/connect/relations.proto";
+import "spark/connect/ml.proto";
 
 package spark.connect;
 
@@ -48,7 +49,7 @@ message Command {
     CheckpointCommand checkpoint_command = 14;
     RemoveCachedRemoteRelationCommand remove_cached_remote_relation_command = 15;
     MergeIntoTableCommand merge_into_table_command = 16;
-
+    MlCommand ml_command = 17;
     // This field is used to mark extensions to the protocol. When plugins generate arbitrary
     // Commands they can add them here. During the planning the correct resolution is done.
     google.protobuf.Any extension = 999;
@@ -507,6 +508,9 @@ message CheckpointCommand {
 
   // (Required) Whether to checkpoint this dataframe immediately.
   bool eager = 3;
+
+  // (Optional) For local checkpoint, the storage level to use.
+  optional StorageLevel storage_level = 4;
 }
 
 message MergeIntoTableCommand {

kyuubi-relocated-spark-connect-rpc/src/main/protobuf/spark/connect/common.proto

@@ -86,6 +86,7 @@ message Origin {
   // (Required) Indicate the origin type.
   oneof function {
     PythonOrigin python_origin = 1;
+    JvmOrigin jvm_origin = 2;
   }
 }
 
@@ -96,3 +97,77 @@ message PythonOrigin {
   // (Required) Callsite to show to end users, for example, stacktrace.
   string call_site = 2;
 }
+
+message JvmOrigin {
+  // (Optional) Line number in the source file.
+  optional int32 line = 1;
+
+  // (Optional) Start position in the source file.
+  optional int32 start_position = 2;
+
+  // (Optional) Start index in the source file.
+  optional int32 start_index = 3;
+
+  // (Optional) Stop index in the source file.
+  optional int32 stop_index = 4;
+
+  // (Optional) SQL text.
+  optional string sql_text = 5;
+
+  // (Optional) Object type.
+  optional string object_type = 6;
+
+  // (Optional) Object name.
+  optional string object_name = 7;
+
+  // (Optional) Stack trace.
+  repeated StackTraceElement stack_trace = 8;
+}
+
+// A message to hold a [[java.lang.StackTraceElement]].
+message StackTraceElement {
+  // (Optional) Class loader name
+  optional string class_loader_name = 1;
+
+  // (Optional) Module name
+  optional string module_name = 2;
+
+  // (Optional) Module version
+  optional string module_version = 3;
+
+  // (Required) Declaring class
+  string declaring_class = 4;
+
+  // (Required) Method name
+  string method_name = 5;
+
+  // (Optional) File name
+  optional string file_name = 6;
+
+  // (Required) Line number
+  int32 line_number = 7;
+}
+
+message Bools {
+  repeated bool values = 1;
+}
+
+message Ints {
+  repeated int32 values = 1;
+}
+
+message Longs {
+  repeated int64 values = 1;
+}
+
+message Floats {
+  repeated float values = 1;
+}
+
+message Doubles {
+  repeated double values = 1;
+}
+
+message Strings {
+  repeated string values = 1;
+}

kyuubi-relocated-spark-connect-rpc/src/main/protobuf/spark/connect/expressions.proto

@@ -52,6 +52,8 @@ message Expression {
     NamedArgumentExpression named_argument_expression = 17;
     MergeAction merge_action = 19;
     TypedAggregateExpression typed_aggregate_expression = 20;
+    LazyExpression lazy_expression = 21;
+    SubqueryExpression subquery_expression = 22;
 
     // This field is used to mark extensions to the protocol. When plugins generate arbitrary
     // relations they can add them here. During the planning the correct resolution is done.
@@ -192,6 +194,8 @@ message Expression {
       Array array = 22;
       Map map = 23;
       Struct struct = 24;
+
+      SpecializedArray specialized_array = 25;
     }
 
     message Decimal {
@@ -226,6 +230,17 @@ message Expression {
       DataType struct_type = 1;
       repeated Literal elements = 2;
     }
+
+    message SpecializedArray {
+      oneof value_type {
+        Bools bools = 1;
+        Ints ints = 2;
+        Longs longs = 3;
+        Floats floats = 4;
+        Doubles doubles = 5;
+        Strings strings = 6;
+      }
+    }
   }
 
   // An unresolved attribute that is not explicitly bound to a specific column, but the column
@@ -259,6 +274,11 @@ message Expression {
     // When it is not a user defined function, Connect will use the function name directly.
     // When it is a user defined function, Connect will parse the function name first.
     bool is_user_defined_function = 4;
+
+    // (Optional) Indicate if this function is defined in the internal function registry.
+    // If not set, the server will try to look up the function in the internal function registry
+    // and decide appropriately.
+    optional bool is_internal = 5;
   }
 
   // Expression as string.
@@ -451,3 +471,22 @@ message MergeAction {
     Expression value = 2;
   }
 }
+
+message LazyExpression {
+  // (Required) The expression to be marked as lazy.
+  Expression child = 1;
+}
+
+message SubqueryExpression {
+  // (Required) The id of corresponding connect plan.
+  int64 plan_id = 1;
+
+  // (Required) The type of the subquery.
+  SubqueryType subquery_type = 2;
+
+  enum SubqueryType {
+    SUBQUERY_TYPE_UNKNOWN = 0;
+    SUBQUERY_TYPE_SCALAR = 1;
+    SUBQUERY_TYPE_EXISTS = 2;
+  }
+}