From e0f3b70002a99ba99d3616ff4f2312c428290110 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 3 Nov 2024 22:47:12 +0800 Subject: [PATCH 01/14] init --- dev/stream-state-gen-protos.sh | 98 ++ dev/tox.ini | 1 + .../sql/streaming/proto/StateMessage_pb2.py | 179 +- .../sql/streaming/proto/StateMessage_pb2.pyi | 1529 ++++++++++++----- sql/core/src/main/buf.gen.yaml | 24 + sql/core/src/main/buf.work.yaml | 19 + 6 files changed, 1332 insertions(+), 518 deletions(-) create mode 100755 dev/stream-state-gen-protos.sh create mode 100644 sql/core/src/main/buf.gen.yaml create mode 100644 sql/core/src/main/buf.work.yaml diff --git a/dev/stream-state-gen-protos.sh b/dev/stream-state-gen-protos.sh new file mode 100755 index 000000000000..48058ee6cf9a --- /dev/null +++ b/dev/stream-state-gen-protos.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -ex + +if [[ $# -gt 0 ]]; then + echo "Illegal number of parameters." + echo "Usage: ./dev/stream-state-gen-protos.sh" + exit -1 +fi + + +SPARK_HOME="$(cd "`dirname $0`"/..; pwd)" +cd "$SPARK_HOME" + + +OUTPUT_PATH=${SPARK_HOME}/python/pyspark/sql/streaming/proto +if [[ $# -eq 1 ]]; then + rm -Rf $1 + mkdir -p $1 + OUTPUT_PATH=$1 +fi + +pushd sql/core/src/main + +LICENSE=$(cat <<'EOF' +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +EOF) +echo "$LICENSE" > /tmp/tmp_licence + + +# Delete the old generated protobuf files. +rm -Rf gen + +# Now, regenerate the new files +buf generate --debug -vvv + +# We need to edit the generate python files to account for the actual package location and not +# the one generated by proto. +for f in `find gen/proto/python -name "*.py*"`; do + # First fix the imports. + if [[ $f == *_pb2.py ]]; then + sed -e 's/from spark.streaming import/from pyspark.sql.streaming.proto import/g' $f > $f.tmp + mv $f.tmp $f + # Now fix the module name in the serialized descriptor. + sed -e "s/DESCRIPTOR, 'spark.streaming/DESCRIPTOR, 'pyspark.sql.streaming.proto/g" $f > $f.tmp + mv $f.tmp $f + elif [[ $f == *.pyi ]]; then + sed -e 's/import spark.streaming./import pyspark.sql.streaming.proto./g' -e 's/spark.streaming./pyspark.sql.streaming.proto./g' -e '/ *@typing_extensions\.final/d' $f > $f.tmp + mv $f.tmp $f + fi + + # Prepend the Apache licence header to the files. + cp $f $f.bak + cat /tmp/tmp_licence $f.bak > $f + + rm $f.bak +done + +black --config $SPARK_HOME/dev/pyproject.toml gen/proto/python + +# Last step copy the result files to the destination module. +for f in `find gen/proto/python -name "*.py*"`; do + cp $f $OUTPUT_PATH +done + +# Clean up everything. +rm -Rf gen diff --git a/dev/tox.ini b/dev/tox.ini index 47b1b4a9d783..05a6b16a03bd 100644 --- a/dev/tox.ini +++ b/dev/tox.ini @@ -59,5 +59,6 @@ exclude = *python/pyspark/worker.pyi, *python/pyspark/java_gateway.pyi, *python/pyspark/sql/connect/proto/*, + *python/pyspark/sql/streaming/proto/*, */venv/* max-line-length = 100 diff --git a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py index aeb195ca10ba..c5cb797a96e5 100644 --- a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py +++ b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py @@ -16,109 +16,104 @@ # # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE -# source: StateMessage.proto -# Protobuf Python Version: 5.27.3 +# source: protobuf/org/apache/spark/sql/execution/streaming/StateMessage.proto """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, 5, 27, 3, "", "StateMessage.proto" -) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x12StateMessage.proto\x12.org.apache.spark.sql.execution.streaming.state"\xbf\x03\n\x0cStateRequest\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x66\n\x15statefulProcessorCall\x18\x02 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.StatefulProcessorCallH\x00\x12\x64\n\x14stateVariableRequest\x18\x03 \x01(\x0b\x32\x44.org.apache.spark.sql.execution.streaming.state.StateVariableRequestH\x00\x12p\n\x1aimplicitGroupingKeyRequest\x18\x04 \x01(\x0b\x32J.org.apache.spark.sql.execution.streaming.state.ImplicitGroupingKeyRequestH\x00\x12T\n\x0ctimerRequest\x18\x05 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.TimerRequestH\x00\x42\x08\n\x06method"H\n\rStateResponse\x12\x12\n\nstatusCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\x0c"W\n\x1cStateResponseWithLongTypeVal\x12\x12\n\nstatusCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\x03"\xea\x03\n\x15StatefulProcessorCall\x12X\n\x0esetHandleState\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetHandleStateH\x00\x12Y\n\rgetValueState\x18\x02 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00\x12X\n\x0cgetListState\x18\x03 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00\x12W\n\x0bgetMapState\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00\x12_\n\x0etimerStateCall\x18\x05 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.TimerStateCallCommandH\x00\x42\x08\n\x06method"\xa8\x02\n\x14StateVariableRequest\x12X\n\x0evalueStateCall\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.ValueStateCallH\x00\x12V\n\rlistStateCall\x18\x02 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.ListStateCallH\x00\x12T\n\x0cmapStateCall\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.MapStateCallH\x00\x42\x08\n\x06method"\xe0\x01\n\x1aImplicitGroupingKeyRequest\x12X\n\x0esetImplicitKey\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetImplicitKeyH\x00\x12^\n\x11removeImplicitKey\x18\x02 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.RemoveImplicitKeyH\x00\x42\x08\n\x06method"\xda\x01\n\x0cTimerRequest\x12^\n\x11timerValueRequest\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.TimerValueRequestH\x00\x12`\n\x12\x65xpiryTimerRequest\x18\x02 \x01(\x0b\x32\x42.org.apache.spark.sql.execution.streaming.state.ExpiryTimerRequestH\x00\x42\x08\n\x06method"\xd4\x01\n\x11TimerValueRequest\x12_\n\x12getProcessingTimer\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.GetProcessingTimeH\x00\x12T\n\x0cgetWatermark\x18\x02 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.GetWatermarkH\x00\x42\x08\n\x06method"/\n\x12\x45xpiryTimerRequest\x12\x19\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03"\x13\n\x11GetProcessingTime"\x0e\n\x0cGetWatermark"\x9a\x01\n\x10StateCallCommand\x12\x11\n\tstateName\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\x1b\n\x13mapStateValueSchema\x18\x03 \x01(\t\x12\x46\n\x03ttl\x18\x04 \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.TTLConfig"\x8f\x02\n\x15TimerStateCallCommand\x12Q\n\x08register\x18\x01 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.RegisterTimerH\x00\x12M\n\x06\x64\x65lete\x18\x02 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.DeleteTimerH\x00\x12J\n\x04list\x18\x03 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.ListTimersH\x00\x42\x08\n\x06method"\xe1\x02\n\x0eValueStateCall\x12\x11\n\tstateName\x18\x01 \x01(\t\x12H\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00\x12\x42\n\x03get\x18\x03 \x01(\x0b\x32\x33.org.apache.spark.sql.execution.streaming.state.GetH\x00\x12\\\n\x10valueStateUpdate\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.ValueStateUpdateH\x00\x12\x46\n\x05\x63lear\x18\x05 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00\x42\x08\n\x06method"\x90\x04\n\rListStateCall\x12\x11\n\tstateName\x18\x01 \x01(\t\x12H\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00\x12T\n\x0clistStateGet\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStateGetH\x00\x12T\n\x0clistStatePut\x18\x04 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStatePutH\x00\x12R\n\x0b\x61ppendValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.AppendValueH\x00\x12P\n\nappendList\x18\x06 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.AppendListH\x00\x12\x46\n\x05\x63lear\x18\x07 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00\x42\x08\n\x06method"\xe1\x05\n\x0cMapStateCall\x12\x11\n\tstateName\x18\x01 \x01(\t\x12H\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00\x12L\n\x08getValue\x18\x03 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.GetValueH\x00\x12R\n\x0b\x63ontainsKey\x18\x04 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.ContainsKeyH\x00\x12R\n\x0bupdateValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.UpdateValueH\x00\x12L\n\x08iterator\x18\x06 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.IteratorH\x00\x12\x44\n\x04keys\x18\x07 \x01(\x0b\x32\x34.org.apache.spark.sql.execution.streaming.state.KeysH\x00\x12H\n\x06values\x18\x08 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ValuesH\x00\x12N\n\tremoveKey\x18\t \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.RemoveKeyH\x00\x12\x46\n\x05\x63lear\x18\n \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00\x42\x08\n\x06method"\x1d\n\x0eSetImplicitKey\x12\x0b\n\x03key\x18\x01 \x01(\x0c"\x13\n\x11RemoveImplicitKey"\x08\n\x06\x45xists"\x05\n\x03Get"*\n\rRegisterTimer\x12\x19\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03"(\n\x0b\x44\x65leteTimer\x12\x19\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03" \n\nListTimers\x12\x12\n\niteratorId\x18\x01 \x01(\t"!\n\x10ValueStateUpdate\x12\r\n\x05value\x18\x01 \x01(\x0c"\x07\n\x05\x43lear""\n\x0cListStateGet\x12\x12\n\niteratorId\x18\x01 \x01(\t"\x0e\n\x0cListStatePut"\x1c\n\x0b\x41ppendValue\x12\r\n\x05value\x18\x01 \x01(\x0c"\x0c\n\nAppendList"\x1b\n\x08GetValue\x12\x0f\n\x07userKey\x18\x01 \x01(\x0c"\x1e\n\x0b\x43ontainsKey\x12\x0f\n\x07userKey\x18\x01 \x01(\x0c"-\n\x0bUpdateValue\x12\x0f\n\x07userKey\x18\x01 \x01(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c"\x1e\n\x08Iterator\x12\x12\n\niteratorId\x18\x01 \x01(\t"\x1a\n\x04Keys\x12\x12\n\niteratorId\x18\x01 \x01(\t"\x1c\n\x06Values\x12\x12\n\niteratorId\x18\x01 \x01(\t"\x1c\n\tRemoveKey\x12\x0f\n\x07userKey\x18\x01 \x01(\x0c"\\\n\x0eSetHandleState\x12J\n\x05state\x18\x01 \x01(\x0e\x32;.org.apache.spark.sql.execution.streaming.state.HandleState"\x1f\n\tTTLConfig\x12\x12\n\ndurationMs\x18\x01 \x01(\x05*`\n\x0bHandleState\x12\x0b\n\x07\x43REATED\x10\x00\x12\x0f\n\x0bINITIALIZED\x10\x01\x12\x12\n\x0e\x44\x41TA_PROCESSED\x10\x02\x12\x13\n\x0fTIMER_PROCESSED\x10\x03\x12\n\n\x06\x43LOSED\x10\x04\x62\x06proto3' # noqa: E501 + b'\nDprotobuf/org/apache/spark/sql/execution/streaming/StateMessage.proto\x12.org.apache.spark.sql.execution.streaming.state"\xa0\x04\n\x0cStateRequest\x12\x18\n\x07version\x18\x01 \x01(\x05R\x07version\x12}\n\x15statefulProcessorCall\x18\x02 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.StatefulProcessorCallH\x00R\x15statefulProcessorCall\x12z\n\x14stateVariableRequest\x18\x03 \x01(\x0b\x32\x44.org.apache.spark.sql.execution.streaming.state.StateVariableRequestH\x00R\x14stateVariableRequest\x12\x8c\x01\n\x1aimplicitGroupingKeyRequest\x18\x04 \x01(\x0b\x32J.org.apache.spark.sql.execution.streaming.state.ImplicitGroupingKeyRequestH\x00R\x1aimplicitGroupingKeyRequest\x12\x62\n\x0ctimerRequest\x18\x05 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.TimerRequestH\x00R\x0ctimerRequestB\x08\n\x06method"i\n\rStateResponse\x12\x1e\n\nstatusCode\x18\x01 \x01(\x05R\nstatusCode\x12"\n\x0c\x65rrorMessage\x18\x02 \x01(\tR\x0c\x65rrorMessage\x12\x14\n\x05value\x18\x03 \x01(\x0cR\x05value"x\n\x1cStateResponseWithLongTypeVal\x12\x1e\n\nstatusCode\x18\x01 \x01(\x05R\nstatusCode\x12"\n\x0c\x65rrorMessage\x18\x02 \x01(\tR\x0c\x65rrorMessage\x12\x14\n\x05value\x18\x03 \x01(\x03R\x05value"\xb4\x04\n\x15StatefulProcessorCall\x12h\n\x0esetHandleState\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetHandleStateH\x00R\x0esetHandleState\x12h\n\rgetValueState\x18\x02 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\rgetValueState\x12\x66\n\x0cgetListState\x18\x03 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0cgetListState\x12\x64\n\x0bgetMapState\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0bgetMapState\x12o\n\x0etimerStateCall\x18\x05 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.TimerStateCallCommandH\x00R\x0etimerStateCallB\x08\n\x06method"\xd5\x02\n\x14StateVariableRequest\x12h\n\x0evalueStateCall\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.ValueStateCallH\x00R\x0evalueStateCall\x12\x65\n\rlistStateCall\x18\x02 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.ListStateCallH\x00R\rlistStateCall\x12\x62\n\x0cmapStateCall\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.MapStateCallH\x00R\x0cmapStateCallB\x08\n\x06method"\x83\x02\n\x1aImplicitGroupingKeyRequest\x12h\n\x0esetImplicitKey\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetImplicitKeyH\x00R\x0esetImplicitKey\x12q\n\x11removeImplicitKey\x18\x02 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.RemoveImplicitKeyH\x00R\x11removeImplicitKeyB\x08\n\x06method"\x81\x02\n\x0cTimerRequest\x12q\n\x11timerValueRequest\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.TimerValueRequestH\x00R\x11timerValueRequest\x12t\n\x12\x65xpiryTimerRequest\x18\x02 \x01(\x0b\x32\x42.org.apache.spark.sql.execution.streaming.state.ExpiryTimerRequestH\x00R\x12\x65xpiryTimerRequestB\x08\n\x06method"\xf6\x01\n\x11TimerValueRequest\x12s\n\x12getProcessingTimer\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.GetProcessingTimeH\x00R\x12getProcessingTimer\x12\x62\n\x0cgetWatermark\x18\x02 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.GetWatermarkH\x00R\x0cgetWatermarkB\x08\n\x06method"B\n\x12\x45xpiryTimerRequest\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs"\x13\n\x11GetProcessingTime"\x0e\n\x0cGetWatermark"\xc7\x01\n\x10StateCallCommand\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12\x16\n\x06schema\x18\x02 \x01(\tR\x06schema\x12\x30\n\x13mapStateValueSchema\x18\x03 \x01(\tR\x13mapStateValueSchema\x12K\n\x03ttl\x18\x04 \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.TTLConfigR\x03ttl"\xa7\x02\n\x15TimerStateCallCommand\x12[\n\x08register\x18\x01 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.RegisterTimerH\x00R\x08register\x12U\n\x06\x64\x65lete\x18\x02 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.DeleteTimerH\x00R\x06\x64\x65lete\x12P\n\x04list\x18\x03 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.ListTimersH\x00R\x04listB\x08\n\x06method"\x92\x03\n\x0eValueStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12G\n\x03get\x18\x03 \x01(\x0b\x32\x33.org.apache.spark.sql.execution.streaming.state.GetH\x00R\x03get\x12n\n\x10valueStateUpdate\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.ValueStateUpdateH\x00R\x10valueStateUpdate\x12M\n\x05\x63lear\x18\x05 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method"\xdf\x04\n\rListStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12\x62\n\x0clistStateGet\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStateGetH\x00R\x0clistStateGet\x12\x62\n\x0clistStatePut\x18\x04 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStatePutH\x00R\x0clistStatePut\x12_\n\x0b\x61ppendValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.AppendValueH\x00R\x0b\x61ppendValue\x12\\\n\nappendList\x18\x06 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.AppendListH\x00R\nappendList\x12M\n\x05\x63lear\x18\x07 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method"\xc2\x06\n\x0cMapStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12V\n\x08getValue\x18\x03 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.GetValueH\x00R\x08getValue\x12_\n\x0b\x63ontainsKey\x18\x04 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.ContainsKeyH\x00R\x0b\x63ontainsKey\x12_\n\x0bupdateValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.UpdateValueH\x00R\x0bupdateValue\x12V\n\x08iterator\x18\x06 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.IteratorH\x00R\x08iterator\x12J\n\x04keys\x18\x07 \x01(\x0b\x32\x34.org.apache.spark.sql.execution.streaming.state.KeysH\x00R\x04keys\x12P\n\x06values\x18\x08 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ValuesH\x00R\x06values\x12Y\n\tremoveKey\x18\t \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.RemoveKeyH\x00R\tremoveKey\x12M\n\x05\x63lear\x18\n \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method""\n\x0eSetImplicitKey\x12\x10\n\x03key\x18\x01 \x01(\x0cR\x03key"\x13\n\x11RemoveImplicitKey"\x08\n\x06\x45xists"\x05\n\x03Get"=\n\rRegisterTimer\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs";\n\x0b\x44\x65leteTimer\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs",\n\nListTimers\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"(\n\x10ValueStateUpdate\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value"\x07\n\x05\x43lear".\n\x0cListStateGet\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"\x0e\n\x0cListStatePut"#\n\x0b\x41ppendValue\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value"\x0c\n\nAppendList"$\n\x08GetValue\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"\'\n\x0b\x43ontainsKey\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"=\n\x0bUpdateValue\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value"*\n\x08Iterator\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"&\n\x04Keys\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"(\n\x06Values\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"%\n\tRemoveKey\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"c\n\x0eSetHandleState\x12Q\n\x05state\x18\x01 \x01(\x0e\x32;.org.apache.spark.sql.execution.streaming.state.HandleStateR\x05state"+\n\tTTLConfig\x12\x1e\n\ndurationMs\x18\x01 \x01(\x05R\ndurationMs*`\n\x0bHandleState\x12\x0b\n\x07\x43REATED\x10\x00\x12\x0f\n\x0bINITIALIZED\x10\x01\x12\x12\n\x0e\x44\x41TA_PROCESSED\x10\x02\x12\x13\n\x0fTIMER_PROCESSED\x10\x03\x12\n\n\x06\x43LOSED\x10\x04\x62\x06proto3' ) -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "StateMessage_pb2", _globals) -if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals["_HANDLESTATE"]._serialized_start = 4966 - _globals["_HANDLESTATE"]._serialized_end = 5062 - _globals["_STATEREQUEST"]._serialized_start = 71 - _globals["_STATEREQUEST"]._serialized_end = 518 - _globals["_STATERESPONSE"]._serialized_start = 520 - _globals["_STATERESPONSE"]._serialized_end = 592 - _globals["_STATERESPONSEWITHLONGTYPEVAL"]._serialized_start = 594 - _globals["_STATERESPONSEWITHLONGTYPEVAL"]._serialized_end = 681 - _globals["_STATEFULPROCESSORCALL"]._serialized_start = 684 - _globals["_STATEFULPROCESSORCALL"]._serialized_end = 1174 - _globals["_STATEVARIABLEREQUEST"]._serialized_start = 1177 - _globals["_STATEVARIABLEREQUEST"]._serialized_end = 1473 - _globals["_IMPLICITGROUPINGKEYREQUEST"]._serialized_start = 1476 - _globals["_IMPLICITGROUPINGKEYREQUEST"]._serialized_end = 1700 - _globals["_TIMERREQUEST"]._serialized_start = 1703 - _globals["_TIMERREQUEST"]._serialized_end = 1921 - _globals["_TIMERVALUEREQUEST"]._serialized_start = 1924 - _globals["_TIMERVALUEREQUEST"]._serialized_end = 2136 - _globals["_EXPIRYTIMERREQUEST"]._serialized_start = 2138 - _globals["_EXPIRYTIMERREQUEST"]._serialized_end = 2185 - _globals["_GETPROCESSINGTIME"]._serialized_start = 2187 - _globals["_GETPROCESSINGTIME"]._serialized_end = 2206 - _globals["_GETWATERMARK"]._serialized_start = 2208 - _globals["_GETWATERMARK"]._serialized_end = 2222 - _globals["_STATECALLCOMMAND"]._serialized_start = 2225 - _globals["_STATECALLCOMMAND"]._serialized_end = 2379 - _globals["_TIMERSTATECALLCOMMAND"]._serialized_start = 2382 - _globals["_TIMERSTATECALLCOMMAND"]._serialized_end = 2653 - _globals["_VALUESTATECALL"]._serialized_start = 2656 - _globals["_VALUESTATECALL"]._serialized_end = 3009 - _globals["_LISTSTATECALL"]._serialized_start = 3012 - _globals["_LISTSTATECALL"]._serialized_end = 3540 - _globals["_MAPSTATECALL"]._serialized_start = 3543 - _globals["_MAPSTATECALL"]._serialized_end = 4280 - _globals["_SETIMPLICITKEY"]._serialized_start = 4282 - _globals["_SETIMPLICITKEY"]._serialized_end = 4311 - _globals["_REMOVEIMPLICITKEY"]._serialized_start = 4313 - _globals["_REMOVEIMPLICITKEY"]._serialized_end = 4332 - _globals["_EXISTS"]._serialized_start = 4334 - _globals["_EXISTS"]._serialized_end = 4342 - _globals["_GET"]._serialized_start = 4344 - _globals["_GET"]._serialized_end = 4349 - _globals["_REGISTERTIMER"]._serialized_start = 4351 - _globals["_REGISTERTIMER"]._serialized_end = 4393 - _globals["_DELETETIMER"]._serialized_start = 4395 - _globals["_DELETETIMER"]._serialized_end = 4435 - _globals["_LISTTIMERS"]._serialized_start = 4437 - _globals["_LISTTIMERS"]._serialized_end = 4469 - _globals["_VALUESTATEUPDATE"]._serialized_start = 4471 - _globals["_VALUESTATEUPDATE"]._serialized_end = 4504 - _globals["_CLEAR"]._serialized_start = 4506 - _globals["_CLEAR"]._serialized_end = 4513 - _globals["_LISTSTATEGET"]._serialized_start = 4515 - _globals["_LISTSTATEGET"]._serialized_end = 4549 - _globals["_LISTSTATEPUT"]._serialized_start = 4551 - _globals["_LISTSTATEPUT"]._serialized_end = 4565 - _globals["_APPENDVALUE"]._serialized_start = 4567 - _globals["_APPENDVALUE"]._serialized_end = 4595 - _globals["_APPENDLIST"]._serialized_start = 4597 - _globals["_APPENDLIST"]._serialized_end = 4609 - _globals["_GETVALUE"]._serialized_start = 4611 - _globals["_GETVALUE"]._serialized_end = 4638 - _globals["_CONTAINSKEY"]._serialized_start = 4640 - _globals["_CONTAINSKEY"]._serialized_end = 4670 - _globals["_UPDATEVALUE"]._serialized_start = 4672 - _globals["_UPDATEVALUE"]._serialized_end = 4717 - _globals["_ITERATOR"]._serialized_start = 4719 - _globals["_ITERATOR"]._serialized_end = 4749 - _globals["_KEYS"]._serialized_start = 4751 - _globals["_KEYS"]._serialized_end = 4777 - _globals["_VALUES"]._serialized_start = 4779 - _globals["_VALUES"]._serialized_end = 4807 - _globals["_REMOVEKEY"]._serialized_start = 4809 - _globals["_REMOVEKEY"]._serialized_end = 4837 - _globals["_SETHANDLESTATE"]._serialized_start = 4839 - _globals["_SETHANDLESTATE"]._serialized_end = 4931 - _globals["_TTLCONFIG"]._serialized_start = 4933 - _globals["_TTLCONFIG"]._serialized_end = 4964 +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages( + DESCRIPTOR, "protobuf.org.apache.spark.sql.execution.streaming.StateMessage_pb2", globals() +) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _HANDLESTATE._serialized_start = 5898 + _HANDLESTATE._serialized_end = 5994 + _STATEREQUEST._serialized_start = 121 + _STATEREQUEST._serialized_end = 665 + _STATERESPONSE._serialized_start = 667 + _STATERESPONSE._serialized_end = 772 + _STATERESPONSEWITHLONGTYPEVAL._serialized_start = 774 + _STATERESPONSEWITHLONGTYPEVAL._serialized_end = 894 + _STATEFULPROCESSORCALL._serialized_start = 897 + _STATEFULPROCESSORCALL._serialized_end = 1461 + _STATEVARIABLEREQUEST._serialized_start = 1464 + _STATEVARIABLEREQUEST._serialized_end = 1805 + _IMPLICITGROUPINGKEYREQUEST._serialized_start = 1808 + _IMPLICITGROUPINGKEYREQUEST._serialized_end = 2067 + _TIMERREQUEST._serialized_start = 2070 + _TIMERREQUEST._serialized_end = 2327 + _TIMERVALUEREQUEST._serialized_start = 2330 + _TIMERVALUEREQUEST._serialized_end = 2576 + _EXPIRYTIMERREQUEST._serialized_start = 2578 + _EXPIRYTIMERREQUEST._serialized_end = 2644 + _GETPROCESSINGTIME._serialized_start = 2646 + _GETPROCESSINGTIME._serialized_end = 2665 + _GETWATERMARK._serialized_start = 2667 + _GETWATERMARK._serialized_end = 2681 + _STATECALLCOMMAND._serialized_start = 2684 + _STATECALLCOMMAND._serialized_end = 2883 + _TIMERSTATECALLCOMMAND._serialized_start = 2886 + _TIMERSTATECALLCOMMAND._serialized_end = 3181 + _VALUESTATECALL._serialized_start = 3184 + _VALUESTATECALL._serialized_end = 3586 + _LISTSTATECALL._serialized_start = 3589 + _LISTSTATECALL._serialized_end = 4196 + _MAPSTATECALL._serialized_start = 4199 + _MAPSTATECALL._serialized_end = 5033 + _SETIMPLICITKEY._serialized_start = 5035 + _SETIMPLICITKEY._serialized_end = 5069 + _REMOVEIMPLICITKEY._serialized_start = 5071 + _REMOVEIMPLICITKEY._serialized_end = 5090 + _EXISTS._serialized_start = 5092 + _EXISTS._serialized_end = 5100 + _GET._serialized_start = 5102 + _GET._serialized_end = 5107 + _REGISTERTIMER._serialized_start = 5109 + _REGISTERTIMER._serialized_end = 5170 + _DELETETIMER._serialized_start = 5172 + _DELETETIMER._serialized_end = 5231 + _LISTTIMERS._serialized_start = 5233 + _LISTTIMERS._serialized_end = 5277 + _VALUESTATEUPDATE._serialized_start = 5279 + _VALUESTATEUPDATE._serialized_end = 5319 + _CLEAR._serialized_start = 5321 + _CLEAR._serialized_end = 5328 + _LISTSTATEGET._serialized_start = 5330 + _LISTSTATEGET._serialized_end = 5376 + _LISTSTATEPUT._serialized_start = 5378 + _LISTSTATEPUT._serialized_end = 5392 + _APPENDVALUE._serialized_start = 5394 + _APPENDVALUE._serialized_end = 5429 + _APPENDLIST._serialized_start = 5431 + _APPENDLIST._serialized_end = 5443 + _GETVALUE._serialized_start = 5445 + _GETVALUE._serialized_end = 5481 + _CONTAINSKEY._serialized_start = 5483 + _CONTAINSKEY._serialized_end = 5522 + _UPDATEVALUE._serialized_start = 5524 + _UPDATEVALUE._serialized_end = 5585 + _ITERATOR._serialized_start = 5587 + _ITERATOR._serialized_end = 5629 + _KEYS._serialized_start = 5631 + _KEYS._serialized_end = 5669 + _VALUES._serialized_start = 5671 + _VALUES._serialized_end = 5711 + _REMOVEKEY._serialized_start = 5713 + _REMOVEKEY._serialized_end = 5750 + _SETHANDLESTATE._serialized_start = 5752 + _SETHANDLESTATE._serialized_end = 5851 + _TTLCONFIG._serialized_start = 5853 + _TTLCONFIG._serialized_end = 5896 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi b/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi index ff525ee136a4..be27da02770f 100644 --- a/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi +++ b/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi @@ -14,429 +14,1106 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ( - ClassVar as _ClassVar, - Mapping as _Mapping, - Optional as _Optional, - Union as _Union, -) - -DESCRIPTOR: _descriptor.FileDescriptor - -class HandleState(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = () - CREATED: _ClassVar[HandleState] - INITIALIZED: _ClassVar[HandleState] - DATA_PROCESSED: _ClassVar[HandleState] - TIMER_PROCESSED: _ClassVar[HandleState] - CLOSED: _ClassVar[HandleState] - -CREATED: HandleState -INITIALIZED: HandleState -DATA_PROCESSED: HandleState -TIMER_PROCESSED: HandleState -CLOSED: HandleState - -class StateRequest(_message.Message): - __slots__ = ( - "version", - "statefulProcessorCall", - "stateVariableRequest", - "implicitGroupingKeyRequest", - "timerRequest", - ) - VERSION_FIELD_NUMBER: _ClassVar[int] - STATEFULPROCESSORCALL_FIELD_NUMBER: _ClassVar[int] - STATEVARIABLEREQUEST_FIELD_NUMBER: _ClassVar[int] - IMPLICITGROUPINGKEYREQUEST_FIELD_NUMBER: _ClassVar[int] - TIMERREQUEST_FIELD_NUMBER: _ClassVar[int] - version: int - statefulProcessorCall: StatefulProcessorCall - stateVariableRequest: StateVariableRequest - implicitGroupingKeyRequest: ImplicitGroupingKeyRequest - timerRequest: TimerRequest - def __init__( - self, - version: _Optional[int] = ..., - statefulProcessorCall: _Optional[_Union[StatefulProcessorCall, _Mapping]] = ..., - stateVariableRequest: _Optional[_Union[StateVariableRequest, _Mapping]] = ..., - implicitGroupingKeyRequest: _Optional[_Union[ImplicitGroupingKeyRequest, _Mapping]] = ..., - timerRequest: _Optional[_Union[TimerRequest, _Mapping]] = ..., - ) -> None: ... - -class StateResponse(_message.Message): - __slots__ = ("statusCode", "errorMessage", "value") - STATUSCODE_FIELD_NUMBER: _ClassVar[int] - ERRORMESSAGE_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - statusCode: int - errorMessage: str - value: bytes - def __init__( - self, - statusCode: _Optional[int] = ..., - errorMessage: _Optional[str] = ..., - value: _Optional[bytes] = ..., - ) -> None: ... - -class StateResponseWithLongTypeVal(_message.Message): - __slots__ = ("statusCode", "errorMessage", "value") - STATUSCODE_FIELD_NUMBER: _ClassVar[int] - ERRORMESSAGE_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - statusCode: int - errorMessage: str - value: int - def __init__( - self, - statusCode: _Optional[int] = ..., - errorMessage: _Optional[str] = ..., - value: _Optional[int] = ..., - ) -> None: ... - -class StatefulProcessorCall(_message.Message): - __slots__ = ("setHandleState", "getValueState", "getListState", "getMapState", "timerStateCall") - SETHANDLESTATE_FIELD_NUMBER: _ClassVar[int] - GETVALUESTATE_FIELD_NUMBER: _ClassVar[int] - GETLISTSTATE_FIELD_NUMBER: _ClassVar[int] - GETMAPSTATE_FIELD_NUMBER: _ClassVar[int] - TIMERSTATECALL_FIELD_NUMBER: _ClassVar[int] - setHandleState: SetHandleState - getValueState: StateCallCommand - getListState: StateCallCommand - getMapState: StateCallCommand - timerStateCall: TimerStateCallCommand - def __init__( - self, - setHandleState: _Optional[_Union[SetHandleState, _Mapping]] = ..., - getValueState: _Optional[_Union[StateCallCommand, _Mapping]] = ..., - getListState: _Optional[_Union[StateCallCommand, _Mapping]] = ..., - getMapState: _Optional[_Union[StateCallCommand, _Mapping]] = ..., - timerStateCall: _Optional[_Union[TimerStateCallCommand, _Mapping]] = ..., - ) -> None: ... - -class StateVariableRequest(_message.Message): - __slots__ = ("valueStateCall", "listStateCall", "mapStateCall") - VALUESTATECALL_FIELD_NUMBER: _ClassVar[int] - LISTSTATECALL_FIELD_NUMBER: _ClassVar[int] - MAPSTATECALL_FIELD_NUMBER: _ClassVar[int] - valueStateCall: ValueStateCall - listStateCall: ListStateCall - mapStateCall: MapStateCall - def __init__( - self, - valueStateCall: _Optional[_Union[ValueStateCall, _Mapping]] = ..., - listStateCall: _Optional[_Union[ListStateCall, _Mapping]] = ..., - mapStateCall: _Optional[_Union[MapStateCall, _Mapping]] = ..., - ) -> None: ... - -class ImplicitGroupingKeyRequest(_message.Message): - __slots__ = ("setImplicitKey", "removeImplicitKey") - SETIMPLICITKEY_FIELD_NUMBER: _ClassVar[int] - REMOVEIMPLICITKEY_FIELD_NUMBER: _ClassVar[int] - setImplicitKey: SetImplicitKey - removeImplicitKey: RemoveImplicitKey - def __init__( - self, - setImplicitKey: _Optional[_Union[SetImplicitKey, _Mapping]] = ..., - removeImplicitKey: _Optional[_Union[RemoveImplicitKey, _Mapping]] = ..., - ) -> None: ... - -class TimerRequest(_message.Message): - __slots__ = ("timerValueRequest", "expiryTimerRequest") - TIMERVALUEREQUEST_FIELD_NUMBER: _ClassVar[int] - EXPIRYTIMERREQUEST_FIELD_NUMBER: _ClassVar[int] - timerValueRequest: TimerValueRequest - expiryTimerRequest: ExpiryTimerRequest - def __init__( - self, - timerValueRequest: _Optional[_Union[TimerValueRequest, _Mapping]] = ..., - expiryTimerRequest: _Optional[_Union[ExpiryTimerRequest, _Mapping]] = ..., - ) -> None: ... - -class TimerValueRequest(_message.Message): - __slots__ = ("getProcessingTimer", "getWatermark") - GETPROCESSINGTIMER_FIELD_NUMBER: _ClassVar[int] - GETWATERMARK_FIELD_NUMBER: _ClassVar[int] - getProcessingTimer: GetProcessingTime - getWatermark: GetWatermark - def __init__( - self, - getProcessingTimer: _Optional[_Union[GetProcessingTime, _Mapping]] = ..., - getWatermark: _Optional[_Union[GetWatermark, _Mapping]] = ..., - ) -> None: ... - -class ExpiryTimerRequest(_message.Message): - __slots__ = ("expiryTimestampMs",) - EXPIRYTIMESTAMPMS_FIELD_NUMBER: _ClassVar[int] - expiryTimestampMs: int - def __init__(self, expiryTimestampMs: _Optional[int] = ...) -> None: ... - -class GetProcessingTime(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class GetWatermark(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class StateCallCommand(_message.Message): - __slots__ = ("stateName", "schema", "mapStateValueSchema", "ttl") - STATENAME_FIELD_NUMBER: _ClassVar[int] - SCHEMA_FIELD_NUMBER: _ClassVar[int] - MAPSTATEVALUESCHEMA_FIELD_NUMBER: _ClassVar[int] - TTL_FIELD_NUMBER: _ClassVar[int] - stateName: str - schema: str - mapStateValueSchema: str - ttl: TTLConfig - def __init__( - self, - stateName: _Optional[str] = ..., - schema: _Optional[str] = ..., - mapStateValueSchema: _Optional[str] = ..., - ttl: _Optional[_Union[TTLConfig, _Mapping]] = ..., - ) -> None: ... - -class TimerStateCallCommand(_message.Message): - __slots__ = ("register", "delete", "list") - REGISTER_FIELD_NUMBER: _ClassVar[int] - DELETE_FIELD_NUMBER: _ClassVar[int] - LIST_FIELD_NUMBER: _ClassVar[int] - register: RegisterTimer - delete: DeleteTimer - list: ListTimers - def __init__( - self, - register: _Optional[_Union[RegisterTimer, _Mapping]] = ..., - delete: _Optional[_Union[DeleteTimer, _Mapping]] = ..., - list: _Optional[_Union[ListTimers, _Mapping]] = ..., - ) -> None: ... - -class ValueStateCall(_message.Message): - __slots__ = ("stateName", "exists", "get", "valueStateUpdate", "clear") - STATENAME_FIELD_NUMBER: _ClassVar[int] - EXISTS_FIELD_NUMBER: _ClassVar[int] - GET_FIELD_NUMBER: _ClassVar[int] - VALUESTATEUPDATE_FIELD_NUMBER: _ClassVar[int] - CLEAR_FIELD_NUMBER: _ClassVar[int] - stateName: str - exists: Exists - get: Get - valueStateUpdate: ValueStateUpdate - clear: Clear - def __init__( - self, - stateName: _Optional[str] = ..., - exists: _Optional[_Union[Exists, _Mapping]] = ..., - get: _Optional[_Union[Get, _Mapping]] = ..., - valueStateUpdate: _Optional[_Union[ValueStateUpdate, _Mapping]] = ..., - clear: _Optional[_Union[Clear, _Mapping]] = ..., - ) -> None: ... - -class ListStateCall(_message.Message): - __slots__ = ( - "stateName", - "exists", - "listStateGet", - "listStatePut", - "appendValue", - "appendList", - "clear", - ) - STATENAME_FIELD_NUMBER: _ClassVar[int] - EXISTS_FIELD_NUMBER: _ClassVar[int] - LISTSTATEGET_FIELD_NUMBER: _ClassVar[int] - LISTSTATEPUT_FIELD_NUMBER: _ClassVar[int] - APPENDVALUE_FIELD_NUMBER: _ClassVar[int] - APPENDLIST_FIELD_NUMBER: _ClassVar[int] - CLEAR_FIELD_NUMBER: _ClassVar[int] - stateName: str - exists: Exists - listStateGet: ListStateGet - listStatePut: ListStatePut - appendValue: AppendValue - appendList: AppendList - clear: Clear - def __init__( - self, - stateName: _Optional[str] = ..., - exists: _Optional[_Union[Exists, _Mapping]] = ..., - listStateGet: _Optional[_Union[ListStateGet, _Mapping]] = ..., - listStatePut: _Optional[_Union[ListStatePut, _Mapping]] = ..., - appendValue: _Optional[_Union[AppendValue, _Mapping]] = ..., - appendList: _Optional[_Union[AppendList, _Mapping]] = ..., - clear: _Optional[_Union[Clear, _Mapping]] = ..., - ) -> None: ... - -class MapStateCall(_message.Message): - __slots__ = ( - "stateName", - "exists", - "getValue", - "containsKey", - "updateValue", - "iterator", - "keys", - "values", - "removeKey", - "clear", - ) - STATENAME_FIELD_NUMBER: _ClassVar[int] - EXISTS_FIELD_NUMBER: _ClassVar[int] - GETVALUE_FIELD_NUMBER: _ClassVar[int] - CONTAINSKEY_FIELD_NUMBER: _ClassVar[int] - UPDATEVALUE_FIELD_NUMBER: _ClassVar[int] - ITERATOR_FIELD_NUMBER: _ClassVar[int] - KEYS_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] - REMOVEKEY_FIELD_NUMBER: _ClassVar[int] - CLEAR_FIELD_NUMBER: _ClassVar[int] - stateName: str - exists: Exists - getValue: GetValue - containsKey: ContainsKey - updateValue: UpdateValue - iterator: Iterator - keys: Keys - values: Values - removeKey: RemoveKey - clear: Clear - def __init__( - self, - stateName: _Optional[str] = ..., - exists: _Optional[_Union[Exists, _Mapping]] = ..., - getValue: _Optional[_Union[GetValue, _Mapping]] = ..., - containsKey: _Optional[_Union[ContainsKey, _Mapping]] = ..., - updateValue: _Optional[_Union[UpdateValue, _Mapping]] = ..., - iterator: _Optional[_Union[Iterator, _Mapping]] = ..., - keys: _Optional[_Union[Keys, _Mapping]] = ..., - values: _Optional[_Union[Values, _Mapping]] = ..., - removeKey: _Optional[_Union[RemoveKey, _Mapping]] = ..., - clear: _Optional[_Union[Clear, _Mapping]] = ..., - ) -> None: ... - -class SetImplicitKey(_message.Message): - __slots__ = ("key",) - KEY_FIELD_NUMBER: _ClassVar[int] - key: bytes - def __init__(self, key: _Optional[bytes] = ...) -> None: ... - -class RemoveImplicitKey(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class Exists(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class Get(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class RegisterTimer(_message.Message): - __slots__ = ("expiryTimestampMs",) - EXPIRYTIMESTAMPMS_FIELD_NUMBER: _ClassVar[int] - expiryTimestampMs: int - def __init__(self, expiryTimestampMs: _Optional[int] = ...) -> None: ... - -class DeleteTimer(_message.Message): - __slots__ = ("expiryTimestampMs",) - EXPIRYTIMESTAMPMS_FIELD_NUMBER: _ClassVar[int] - expiryTimestampMs: int - def __init__(self, expiryTimestampMs: _Optional[int] = ...) -> None: ... - -class ListTimers(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class ValueStateUpdate(_message.Message): - __slots__ = ("value",) - VALUE_FIELD_NUMBER: _ClassVar[int] - value: bytes - def __init__(self, value: _Optional[bytes] = ...) -> None: ... - -class Clear(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class ListStateGet(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class ListStatePut(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class AppendValue(_message.Message): - __slots__ = ("value",) - VALUE_FIELD_NUMBER: _ClassVar[int] - value: bytes - def __init__(self, value: _Optional[bytes] = ...) -> None: ... - -class AppendList(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class GetValue(_message.Message): - __slots__ = ("userKey",) - USERKEY_FIELD_NUMBER: _ClassVar[int] - userKey: bytes - def __init__(self, userKey: _Optional[bytes] = ...) -> None: ... - -class ContainsKey(_message.Message): - __slots__ = ("userKey",) - USERKEY_FIELD_NUMBER: _ClassVar[int] - userKey: bytes - def __init__(self, userKey: _Optional[bytes] = ...) -> None: ... - -class UpdateValue(_message.Message): - __slots__ = ("userKey", "value") - USERKEY_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - userKey: bytes - value: bytes - def __init__(self, userKey: _Optional[bytes] = ..., value: _Optional[bytes] = ...) -> None: ... - -class Iterator(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class Keys(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class Values(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class RemoveKey(_message.Message): - __slots__ = ("userKey",) - USERKEY_FIELD_NUMBER: _ClassVar[int] - userKey: bytes - def __init__(self, userKey: _Optional[bytes] = ...) -> None: ... - -class SetHandleState(_message.Message): - __slots__ = ("state",) - STATE_FIELD_NUMBER: _ClassVar[int] - state: HandleState - def __init__(self, state: _Optional[_Union[HandleState, str]] = ...) -> None: ... - -class TTLConfig(_message.Message): - __slots__ = ("durationMs",) - DURATIONMS_FIELD_NUMBER: _ClassVar[int] - durationMs: int - def __init__(self, durationMs: _Optional[int] = ...) -> None: ... +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _HandleState: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _HandleStateEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_HandleState.ValueType], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + CREATED: _HandleState.ValueType # 0 + INITIALIZED: _HandleState.ValueType # 1 + DATA_PROCESSED: _HandleState.ValueType # 2 + TIMER_PROCESSED: _HandleState.ValueType # 3 + CLOSED: _HandleState.ValueType # 4 + +class HandleState(_HandleState, metaclass=_HandleStateEnumTypeWrapper): ... + +CREATED: HandleState.ValueType # 0 +INITIALIZED: HandleState.ValueType # 1 +DATA_PROCESSED: HandleState.ValueType # 2 +TIMER_PROCESSED: HandleState.ValueType # 3 +CLOSED: HandleState.ValueType # 4 +global___HandleState = HandleState + +class StateRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VERSION_FIELD_NUMBER: builtins.int + STATEFULPROCESSORCALL_FIELD_NUMBER: builtins.int + STATEVARIABLEREQUEST_FIELD_NUMBER: builtins.int + IMPLICITGROUPINGKEYREQUEST_FIELD_NUMBER: builtins.int + TIMERREQUEST_FIELD_NUMBER: builtins.int + version: builtins.int + @property + def statefulProcessorCall(self) -> global___StatefulProcessorCall: ... + @property + def stateVariableRequest(self) -> global___StateVariableRequest: ... + @property + def implicitGroupingKeyRequest(self) -> global___ImplicitGroupingKeyRequest: ... + @property + def timerRequest(self) -> global___TimerRequest: ... + def __init__( + self, + *, + version: builtins.int = ..., + statefulProcessorCall: global___StatefulProcessorCall | None = ..., + stateVariableRequest: global___StateVariableRequest | None = ..., + implicitGroupingKeyRequest: global___ImplicitGroupingKeyRequest | None = ..., + timerRequest: global___TimerRequest | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "implicitGroupingKeyRequest", + b"implicitGroupingKeyRequest", + "method", + b"method", + "stateVariableRequest", + b"stateVariableRequest", + "statefulProcessorCall", + b"statefulProcessorCall", + "timerRequest", + b"timerRequest", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "implicitGroupingKeyRequest", + b"implicitGroupingKeyRequest", + "method", + b"method", + "stateVariableRequest", + b"stateVariableRequest", + "statefulProcessorCall", + b"statefulProcessorCall", + "timerRequest", + b"timerRequest", + "version", + b"version", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> ( + typing_extensions.Literal[ + "statefulProcessorCall", + "stateVariableRequest", + "implicitGroupingKeyRequest", + "timerRequest", + ] + | None + ): ... + +global___StateRequest = StateRequest + +class StateResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATUSCODE_FIELD_NUMBER: builtins.int + ERRORMESSAGE_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + statusCode: builtins.int + errorMessage: builtins.str + value: builtins.bytes + def __init__( + self, + *, + statusCode: builtins.int = ..., + errorMessage: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "errorMessage", b"errorMessage", "statusCode", b"statusCode", "value", b"value" + ], + ) -> None: ... + +global___StateResponse = StateResponse + +class StateResponseWithLongTypeVal(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATUSCODE_FIELD_NUMBER: builtins.int + ERRORMESSAGE_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + statusCode: builtins.int + errorMessage: builtins.str + value: builtins.int + def __init__( + self, + *, + statusCode: builtins.int = ..., + errorMessage: builtins.str = ..., + value: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "errorMessage", b"errorMessage", "statusCode", b"statusCode", "value", b"value" + ], + ) -> None: ... + +global___StateResponseWithLongTypeVal = StateResponseWithLongTypeVal + +class StatefulProcessorCall(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SETHANDLESTATE_FIELD_NUMBER: builtins.int + GETVALUESTATE_FIELD_NUMBER: builtins.int + GETLISTSTATE_FIELD_NUMBER: builtins.int + GETMAPSTATE_FIELD_NUMBER: builtins.int + TIMERSTATECALL_FIELD_NUMBER: builtins.int + @property + def setHandleState(self) -> global___SetHandleState: ... + @property + def getValueState(self) -> global___StateCallCommand: ... + @property + def getListState(self) -> global___StateCallCommand: ... + @property + def getMapState(self) -> global___StateCallCommand: ... + @property + def timerStateCall(self) -> global___TimerStateCallCommand: ... + def __init__( + self, + *, + setHandleState: global___SetHandleState | None = ..., + getValueState: global___StateCallCommand | None = ..., + getListState: global___StateCallCommand | None = ..., + getMapState: global___StateCallCommand | None = ..., + timerStateCall: global___TimerStateCallCommand | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "getListState", + b"getListState", + "getMapState", + b"getMapState", + "getValueState", + b"getValueState", + "method", + b"method", + "setHandleState", + b"setHandleState", + "timerStateCall", + b"timerStateCall", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "getListState", + b"getListState", + "getMapState", + b"getMapState", + "getValueState", + b"getValueState", + "method", + b"method", + "setHandleState", + b"setHandleState", + "timerStateCall", + b"timerStateCall", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> ( + typing_extensions.Literal[ + "setHandleState", "getValueState", "getListState", "getMapState", "timerStateCall" + ] + | None + ): ... + +global___StatefulProcessorCall = StatefulProcessorCall + +class StateVariableRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VALUESTATECALL_FIELD_NUMBER: builtins.int + LISTSTATECALL_FIELD_NUMBER: builtins.int + MAPSTATECALL_FIELD_NUMBER: builtins.int + @property + def valueStateCall(self) -> global___ValueStateCall: ... + @property + def listStateCall(self) -> global___ListStateCall: ... + @property + def mapStateCall(self) -> global___MapStateCall: ... + def __init__( + self, + *, + valueStateCall: global___ValueStateCall | None = ..., + listStateCall: global___ListStateCall | None = ..., + mapStateCall: global___MapStateCall | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "listStateCall", + b"listStateCall", + "mapStateCall", + b"mapStateCall", + "method", + b"method", + "valueStateCall", + b"valueStateCall", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "listStateCall", + b"listStateCall", + "mapStateCall", + b"mapStateCall", + "method", + b"method", + "valueStateCall", + b"valueStateCall", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["valueStateCall", "listStateCall", "mapStateCall"] | None: ... + +global___StateVariableRequest = StateVariableRequest + +class ImplicitGroupingKeyRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SETIMPLICITKEY_FIELD_NUMBER: builtins.int + REMOVEIMPLICITKEY_FIELD_NUMBER: builtins.int + @property + def setImplicitKey(self) -> global___SetImplicitKey: ... + @property + def removeImplicitKey(self) -> global___RemoveImplicitKey: ... + def __init__( + self, + *, + setImplicitKey: global___SetImplicitKey | None = ..., + removeImplicitKey: global___RemoveImplicitKey | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "method", + b"method", + "removeImplicitKey", + b"removeImplicitKey", + "setImplicitKey", + b"setImplicitKey", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "method", + b"method", + "removeImplicitKey", + b"removeImplicitKey", + "setImplicitKey", + b"setImplicitKey", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["setImplicitKey", "removeImplicitKey"] | None: ... + +global___ImplicitGroupingKeyRequest = ImplicitGroupingKeyRequest + +class TimerRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TIMERVALUEREQUEST_FIELD_NUMBER: builtins.int + EXPIRYTIMERREQUEST_FIELD_NUMBER: builtins.int + @property + def timerValueRequest(self) -> global___TimerValueRequest: ... + @property + def expiryTimerRequest(self) -> global___ExpiryTimerRequest: ... + def __init__( + self, + *, + timerValueRequest: global___TimerValueRequest | None = ..., + expiryTimerRequest: global___ExpiryTimerRequest | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "expiryTimerRequest", + b"expiryTimerRequest", + "method", + b"method", + "timerValueRequest", + b"timerValueRequest", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "expiryTimerRequest", + b"expiryTimerRequest", + "method", + b"method", + "timerValueRequest", + b"timerValueRequest", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["timerValueRequest", "expiryTimerRequest"] | None: ... + +global___TimerRequest = TimerRequest + +class TimerValueRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + GETPROCESSINGTIMER_FIELD_NUMBER: builtins.int + GETWATERMARK_FIELD_NUMBER: builtins.int + @property + def getProcessingTimer(self) -> global___GetProcessingTime: ... + @property + def getWatermark(self) -> global___GetWatermark: ... + def __init__( + self, + *, + getProcessingTimer: global___GetProcessingTime | None = ..., + getWatermark: global___GetWatermark | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "getProcessingTimer", + b"getProcessingTimer", + "getWatermark", + b"getWatermark", + "method", + b"method", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "getProcessingTimer", + b"getProcessingTimer", + "getWatermark", + b"getWatermark", + "method", + b"method", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["getProcessingTimer", "getWatermark"] | None: ... + +global___TimerValueRequest = TimerValueRequest + +class ExpiryTimerRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EXPIRYTIMESTAMPMS_FIELD_NUMBER: builtins.int + expiryTimestampMs: builtins.int + def __init__( + self, + *, + expiryTimestampMs: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["expiryTimestampMs", b"expiryTimestampMs"] + ) -> None: ... + +global___ExpiryTimerRequest = ExpiryTimerRequest + +class GetProcessingTime(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___GetProcessingTime = GetProcessingTime + +class GetWatermark(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___GetWatermark = GetWatermark + +class StateCallCommand(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATENAME_FIELD_NUMBER: builtins.int + SCHEMA_FIELD_NUMBER: builtins.int + MAPSTATEVALUESCHEMA_FIELD_NUMBER: builtins.int + TTL_FIELD_NUMBER: builtins.int + stateName: builtins.str + schema: builtins.str + mapStateValueSchema: builtins.str + @property + def ttl(self) -> global___TTLConfig: ... + def __init__( + self, + *, + stateName: builtins.str = ..., + schema: builtins.str = ..., + mapStateValueSchema: builtins.str = ..., + ttl: global___TTLConfig | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["ttl", b"ttl"]) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "mapStateValueSchema", + b"mapStateValueSchema", + "schema", + b"schema", + "stateName", + b"stateName", + "ttl", + b"ttl", + ], + ) -> None: ... + +global___StateCallCommand = StateCallCommand + +class TimerStateCallCommand(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REGISTER_FIELD_NUMBER: builtins.int + DELETE_FIELD_NUMBER: builtins.int + LIST_FIELD_NUMBER: builtins.int + @property + def register(self) -> global___RegisterTimer: ... + @property + def delete(self) -> global___DeleteTimer: ... + @property + def list(self) -> global___ListTimers: ... + def __init__( + self, + *, + register: global___RegisterTimer | None = ..., + delete: global___DeleteTimer | None = ..., + list: global___ListTimers | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "delete", b"delete", "list", b"list", "method", b"method", "register", b"register" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "delete", b"delete", "list", b"list", "method", b"method", "register", b"register" + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["register", "delete", "list"] | None: ... + +global___TimerStateCallCommand = TimerStateCallCommand + +class ValueStateCall(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATENAME_FIELD_NUMBER: builtins.int + EXISTS_FIELD_NUMBER: builtins.int + GET_FIELD_NUMBER: builtins.int + VALUESTATEUPDATE_FIELD_NUMBER: builtins.int + CLEAR_FIELD_NUMBER: builtins.int + stateName: builtins.str + @property + def exists(self) -> global___Exists: ... + @property + def get(self) -> global___Get: ... + @property + def valueStateUpdate(self) -> global___ValueStateUpdate: ... + @property + def clear(self) -> global___Clear: ... + def __init__( + self, + *, + stateName: builtins.str = ..., + exists: global___Exists | None = ..., + get: global___Get | None = ..., + valueStateUpdate: global___ValueStateUpdate | None = ..., + clear: global___Clear | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "clear", + b"clear", + "exists", + b"exists", + "get", + b"get", + "method", + b"method", + "valueStateUpdate", + b"valueStateUpdate", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "clear", + b"clear", + "exists", + b"exists", + "get", + b"get", + "method", + b"method", + "stateName", + b"stateName", + "valueStateUpdate", + b"valueStateUpdate", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["exists", "get", "valueStateUpdate", "clear"] | None: ... + +global___ValueStateCall = ValueStateCall + +class ListStateCall(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATENAME_FIELD_NUMBER: builtins.int + EXISTS_FIELD_NUMBER: builtins.int + LISTSTATEGET_FIELD_NUMBER: builtins.int + LISTSTATEPUT_FIELD_NUMBER: builtins.int + APPENDVALUE_FIELD_NUMBER: builtins.int + APPENDLIST_FIELD_NUMBER: builtins.int + CLEAR_FIELD_NUMBER: builtins.int + stateName: builtins.str + @property + def exists(self) -> global___Exists: ... + @property + def listStateGet(self) -> global___ListStateGet: ... + @property + def listStatePut(self) -> global___ListStatePut: ... + @property + def appendValue(self) -> global___AppendValue: ... + @property + def appendList(self) -> global___AppendList: ... + @property + def clear(self) -> global___Clear: ... + def __init__( + self, + *, + stateName: builtins.str = ..., + exists: global___Exists | None = ..., + listStateGet: global___ListStateGet | None = ..., + listStatePut: global___ListStatePut | None = ..., + appendValue: global___AppendValue | None = ..., + appendList: global___AppendList | None = ..., + clear: global___Clear | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "appendList", + b"appendList", + "appendValue", + b"appendValue", + "clear", + b"clear", + "exists", + b"exists", + "listStateGet", + b"listStateGet", + "listStatePut", + b"listStatePut", + "method", + b"method", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "appendList", + b"appendList", + "appendValue", + b"appendValue", + "clear", + b"clear", + "exists", + b"exists", + "listStateGet", + b"listStateGet", + "listStatePut", + b"listStatePut", + "method", + b"method", + "stateName", + b"stateName", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> ( + typing_extensions.Literal[ + "exists", "listStateGet", "listStatePut", "appendValue", "appendList", "clear" + ] + | None + ): ... + +global___ListStateCall = ListStateCall + +class MapStateCall(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATENAME_FIELD_NUMBER: builtins.int + EXISTS_FIELD_NUMBER: builtins.int + GETVALUE_FIELD_NUMBER: builtins.int + CONTAINSKEY_FIELD_NUMBER: builtins.int + UPDATEVALUE_FIELD_NUMBER: builtins.int + ITERATOR_FIELD_NUMBER: builtins.int + KEYS_FIELD_NUMBER: builtins.int + VALUES_FIELD_NUMBER: builtins.int + REMOVEKEY_FIELD_NUMBER: builtins.int + CLEAR_FIELD_NUMBER: builtins.int + stateName: builtins.str + @property + def exists(self) -> global___Exists: ... + @property + def getValue(self) -> global___GetValue: ... + @property + def containsKey(self) -> global___ContainsKey: ... + @property + def updateValue(self) -> global___UpdateValue: ... + @property + def iterator(self) -> global___Iterator: ... + @property + def keys(self) -> global___Keys: ... + @property + def values(self) -> global___Values: ... + @property + def removeKey(self) -> global___RemoveKey: ... + @property + def clear(self) -> global___Clear: ... + def __init__( + self, + *, + stateName: builtins.str = ..., + exists: global___Exists | None = ..., + getValue: global___GetValue | None = ..., + containsKey: global___ContainsKey | None = ..., + updateValue: global___UpdateValue | None = ..., + iterator: global___Iterator | None = ..., + keys: global___Keys | None = ..., + values: global___Values | None = ..., + removeKey: global___RemoveKey | None = ..., + clear: global___Clear | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "clear", + b"clear", + "containsKey", + b"containsKey", + "exists", + b"exists", + "getValue", + b"getValue", + "iterator", + b"iterator", + "keys", + b"keys", + "method", + b"method", + "removeKey", + b"removeKey", + "updateValue", + b"updateValue", + "values", + b"values", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "clear", + b"clear", + "containsKey", + b"containsKey", + "exists", + b"exists", + "getValue", + b"getValue", + "iterator", + b"iterator", + "keys", + b"keys", + "method", + b"method", + "removeKey", + b"removeKey", + "stateName", + b"stateName", + "updateValue", + b"updateValue", + "values", + b"values", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> ( + typing_extensions.Literal[ + "exists", + "getValue", + "containsKey", + "updateValue", + "iterator", + "keys", + "values", + "removeKey", + "clear", + ] + | None + ): ... + +global___MapStateCall = MapStateCall + +class SetImplicitKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + key: builtins.bytes + def __init__( + self, + *, + key: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key"]) -> None: ... + +global___SetImplicitKey = SetImplicitKey + +class RemoveImplicitKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___RemoveImplicitKey = RemoveImplicitKey + +class Exists(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___Exists = Exists + +class Get(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___Get = Get + +class RegisterTimer(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EXPIRYTIMESTAMPMS_FIELD_NUMBER: builtins.int + expiryTimestampMs: builtins.int + def __init__( + self, + *, + expiryTimestampMs: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["expiryTimestampMs", b"expiryTimestampMs"] + ) -> None: ... + +global___RegisterTimer = RegisterTimer + +class DeleteTimer(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EXPIRYTIMESTAMPMS_FIELD_NUMBER: builtins.int + expiryTimestampMs: builtins.int + def __init__( + self, + *, + expiryTimestampMs: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["expiryTimestampMs", b"expiryTimestampMs"] + ) -> None: ... + +global___DeleteTimer = DeleteTimer + +class ListTimers(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___ListTimers = ListTimers + +class ValueStateUpdate(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VALUE_FIELD_NUMBER: builtins.int + value: builtins.bytes + def __init__( + self, + *, + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ... + +global___ValueStateUpdate = ValueStateUpdate + +class Clear(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___Clear = Clear + +class ListStateGet(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___ListStateGet = ListStateGet + +class ListStatePut(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___ListStatePut = ListStatePut + +class AppendValue(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VALUE_FIELD_NUMBER: builtins.int + value: builtins.bytes + def __init__( + self, + *, + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ... + +global___AppendValue = AppendValue + +class AppendList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___AppendList = AppendList + +class GetValue(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + USERKEY_FIELD_NUMBER: builtins.int + userKey: builtins.bytes + def __init__( + self, + *, + userKey: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["userKey", b"userKey"]) -> None: ... + +global___GetValue = GetValue + +class ContainsKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + USERKEY_FIELD_NUMBER: builtins.int + userKey: builtins.bytes + def __init__( + self, + *, + userKey: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["userKey", b"userKey"]) -> None: ... + +global___ContainsKey = ContainsKey + +class UpdateValue(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + USERKEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + userKey: builtins.bytes + value: builtins.bytes + def __init__( + self, + *, + userKey: builtins.bytes = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["userKey", b"userKey", "value", b"value"] + ) -> None: ... + +global___UpdateValue = UpdateValue + +class Iterator(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___Iterator = Iterator + +class Keys(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___Keys = Keys + +class Values(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___Values = Values + +class RemoveKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + USERKEY_FIELD_NUMBER: builtins.int + userKey: builtins.bytes + def __init__( + self, + *, + userKey: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["userKey", b"userKey"]) -> None: ... + +global___RemoveKey = RemoveKey + +class SetHandleState(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATE_FIELD_NUMBER: builtins.int + state: global___HandleState.ValueType + def __init__( + self, + *, + state: global___HandleState.ValueType = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["state", b"state"]) -> None: ... + +global___SetHandleState = SetHandleState + +class TTLConfig(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DURATIONMS_FIELD_NUMBER: builtins.int + durationMs: builtins.int + def __init__( + self, + *, + durationMs: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["durationMs", b"durationMs"] + ) -> None: ... + +global___TTLConfig = TTLConfig diff --git a/sql/core/src/main/buf.gen.yaml b/sql/core/src/main/buf.gen.yaml new file mode 100644 index 000000000000..d07abcca6ecc --- /dev/null +++ b/sql/core/src/main/buf.gen.yaml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: v1 +plugins: + # Building the Python build and building the mypy interfaces. + - plugin: buf.build/protocolbuffers/python:v21.7 + out: gen/proto/python + - name: mypy + out: gen/proto/python + diff --git a/sql/core/src/main/buf.work.yaml b/sql/core/src/main/buf.work.yaml new file mode 100644 index 000000000000..a02dead420cd --- /dev/null +++ b/sql/core/src/main/buf.work.yaml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: v1 +directories: + - protobuf From 6cf4a24c7b95b1883d939463ce14aa002517dd04 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 5 Nov 2024 13:42:25 +0800 Subject: [PATCH 02/14] use v27.3 --- .../sql/streaming/proto/StateMessage_pb2.py | 182 ++++++++++-------- sql/core/src/main/buf.gen.yaml | 2 +- 2 files changed, 98 insertions(+), 86 deletions(-) diff --git a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py index c5cb797a96e5..fd175cb3a48a 100644 --- a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py +++ b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py @@ -16,104 +16,116 @@ # # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: protobuf/org/apache/spark/sql/execution/streaming/StateMessage.proto +# NO CHECKED-IN PROTOBUF GENCODE +# source: org/apache/spark/sql/execution/streaming/StateMessage.proto +# Protobuf Python Version: 5.27.3 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 5, + 27, + 3, + "", + "org/apache/spark/sql/execution/streaming/StateMessage.proto", +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\nDprotobuf/org/apache/spark/sql/execution/streaming/StateMessage.proto\x12.org.apache.spark.sql.execution.streaming.state"\xa0\x04\n\x0cStateRequest\x12\x18\n\x07version\x18\x01 \x01(\x05R\x07version\x12}\n\x15statefulProcessorCall\x18\x02 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.StatefulProcessorCallH\x00R\x15statefulProcessorCall\x12z\n\x14stateVariableRequest\x18\x03 \x01(\x0b\x32\x44.org.apache.spark.sql.execution.streaming.state.StateVariableRequestH\x00R\x14stateVariableRequest\x12\x8c\x01\n\x1aimplicitGroupingKeyRequest\x18\x04 \x01(\x0b\x32J.org.apache.spark.sql.execution.streaming.state.ImplicitGroupingKeyRequestH\x00R\x1aimplicitGroupingKeyRequest\x12\x62\n\x0ctimerRequest\x18\x05 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.TimerRequestH\x00R\x0ctimerRequestB\x08\n\x06method"i\n\rStateResponse\x12\x1e\n\nstatusCode\x18\x01 \x01(\x05R\nstatusCode\x12"\n\x0c\x65rrorMessage\x18\x02 \x01(\tR\x0c\x65rrorMessage\x12\x14\n\x05value\x18\x03 \x01(\x0cR\x05value"x\n\x1cStateResponseWithLongTypeVal\x12\x1e\n\nstatusCode\x18\x01 \x01(\x05R\nstatusCode\x12"\n\x0c\x65rrorMessage\x18\x02 \x01(\tR\x0c\x65rrorMessage\x12\x14\n\x05value\x18\x03 \x01(\x03R\x05value"\xb4\x04\n\x15StatefulProcessorCall\x12h\n\x0esetHandleState\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetHandleStateH\x00R\x0esetHandleState\x12h\n\rgetValueState\x18\x02 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\rgetValueState\x12\x66\n\x0cgetListState\x18\x03 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0cgetListState\x12\x64\n\x0bgetMapState\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0bgetMapState\x12o\n\x0etimerStateCall\x18\x05 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.TimerStateCallCommandH\x00R\x0etimerStateCallB\x08\n\x06method"\xd5\x02\n\x14StateVariableRequest\x12h\n\x0evalueStateCall\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.ValueStateCallH\x00R\x0evalueStateCall\x12\x65\n\rlistStateCall\x18\x02 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.ListStateCallH\x00R\rlistStateCall\x12\x62\n\x0cmapStateCall\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.MapStateCallH\x00R\x0cmapStateCallB\x08\n\x06method"\x83\x02\n\x1aImplicitGroupingKeyRequest\x12h\n\x0esetImplicitKey\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetImplicitKeyH\x00R\x0esetImplicitKey\x12q\n\x11removeImplicitKey\x18\x02 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.RemoveImplicitKeyH\x00R\x11removeImplicitKeyB\x08\n\x06method"\x81\x02\n\x0cTimerRequest\x12q\n\x11timerValueRequest\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.TimerValueRequestH\x00R\x11timerValueRequest\x12t\n\x12\x65xpiryTimerRequest\x18\x02 \x01(\x0b\x32\x42.org.apache.spark.sql.execution.streaming.state.ExpiryTimerRequestH\x00R\x12\x65xpiryTimerRequestB\x08\n\x06method"\xf6\x01\n\x11TimerValueRequest\x12s\n\x12getProcessingTimer\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.GetProcessingTimeH\x00R\x12getProcessingTimer\x12\x62\n\x0cgetWatermark\x18\x02 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.GetWatermarkH\x00R\x0cgetWatermarkB\x08\n\x06method"B\n\x12\x45xpiryTimerRequest\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs"\x13\n\x11GetProcessingTime"\x0e\n\x0cGetWatermark"\xc7\x01\n\x10StateCallCommand\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12\x16\n\x06schema\x18\x02 \x01(\tR\x06schema\x12\x30\n\x13mapStateValueSchema\x18\x03 \x01(\tR\x13mapStateValueSchema\x12K\n\x03ttl\x18\x04 \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.TTLConfigR\x03ttl"\xa7\x02\n\x15TimerStateCallCommand\x12[\n\x08register\x18\x01 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.RegisterTimerH\x00R\x08register\x12U\n\x06\x64\x65lete\x18\x02 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.DeleteTimerH\x00R\x06\x64\x65lete\x12P\n\x04list\x18\x03 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.ListTimersH\x00R\x04listB\x08\n\x06method"\x92\x03\n\x0eValueStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12G\n\x03get\x18\x03 \x01(\x0b\x32\x33.org.apache.spark.sql.execution.streaming.state.GetH\x00R\x03get\x12n\n\x10valueStateUpdate\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.ValueStateUpdateH\x00R\x10valueStateUpdate\x12M\n\x05\x63lear\x18\x05 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method"\xdf\x04\n\rListStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12\x62\n\x0clistStateGet\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStateGetH\x00R\x0clistStateGet\x12\x62\n\x0clistStatePut\x18\x04 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStatePutH\x00R\x0clistStatePut\x12_\n\x0b\x61ppendValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.AppendValueH\x00R\x0b\x61ppendValue\x12\\\n\nappendList\x18\x06 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.AppendListH\x00R\nappendList\x12M\n\x05\x63lear\x18\x07 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method"\xc2\x06\n\x0cMapStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12V\n\x08getValue\x18\x03 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.GetValueH\x00R\x08getValue\x12_\n\x0b\x63ontainsKey\x18\x04 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.ContainsKeyH\x00R\x0b\x63ontainsKey\x12_\n\x0bupdateValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.UpdateValueH\x00R\x0bupdateValue\x12V\n\x08iterator\x18\x06 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.IteratorH\x00R\x08iterator\x12J\n\x04keys\x18\x07 \x01(\x0b\x32\x34.org.apache.spark.sql.execution.streaming.state.KeysH\x00R\x04keys\x12P\n\x06values\x18\x08 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ValuesH\x00R\x06values\x12Y\n\tremoveKey\x18\t \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.RemoveKeyH\x00R\tremoveKey\x12M\n\x05\x63lear\x18\n \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method""\n\x0eSetImplicitKey\x12\x10\n\x03key\x18\x01 \x01(\x0cR\x03key"\x13\n\x11RemoveImplicitKey"\x08\n\x06\x45xists"\x05\n\x03Get"=\n\rRegisterTimer\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs";\n\x0b\x44\x65leteTimer\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs",\n\nListTimers\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"(\n\x10ValueStateUpdate\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value"\x07\n\x05\x43lear".\n\x0cListStateGet\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"\x0e\n\x0cListStatePut"#\n\x0b\x41ppendValue\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value"\x0c\n\nAppendList"$\n\x08GetValue\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"\'\n\x0b\x43ontainsKey\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"=\n\x0bUpdateValue\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value"*\n\x08Iterator\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"&\n\x04Keys\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"(\n\x06Values\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"%\n\tRemoveKey\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"c\n\x0eSetHandleState\x12Q\n\x05state\x18\x01 \x01(\x0e\x32;.org.apache.spark.sql.execution.streaming.state.HandleStateR\x05state"+\n\tTTLConfig\x12\x1e\n\ndurationMs\x18\x01 \x01(\x05R\ndurationMs*`\n\x0bHandleState\x12\x0b\n\x07\x43REATED\x10\x00\x12\x0f\n\x0bINITIALIZED\x10\x01\x12\x12\n\x0e\x44\x41TA_PROCESSED\x10\x02\x12\x13\n\x0fTIMER_PROCESSED\x10\x03\x12\n\n\x06\x43LOSED\x10\x04\x62\x06proto3' + b'\n;org/apache/spark/sql/execution/streaming/StateMessage.proto\x12.org.apache.spark.sql.execution.streaming.state"\xa0\x04\n\x0cStateRequest\x12\x18\n\x07version\x18\x01 \x01(\x05R\x07version\x12}\n\x15statefulProcessorCall\x18\x02 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.StatefulProcessorCallH\x00R\x15statefulProcessorCall\x12z\n\x14stateVariableRequest\x18\x03 \x01(\x0b\x32\x44.org.apache.spark.sql.execution.streaming.state.StateVariableRequestH\x00R\x14stateVariableRequest\x12\x8c\x01\n\x1aimplicitGroupingKeyRequest\x18\x04 \x01(\x0b\x32J.org.apache.spark.sql.execution.streaming.state.ImplicitGroupingKeyRequestH\x00R\x1aimplicitGroupingKeyRequest\x12\x62\n\x0ctimerRequest\x18\x05 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.TimerRequestH\x00R\x0ctimerRequestB\x08\n\x06method"i\n\rStateResponse\x12\x1e\n\nstatusCode\x18\x01 \x01(\x05R\nstatusCode\x12"\n\x0c\x65rrorMessage\x18\x02 \x01(\tR\x0c\x65rrorMessage\x12\x14\n\x05value\x18\x03 \x01(\x0cR\x05value"x\n\x1cStateResponseWithLongTypeVal\x12\x1e\n\nstatusCode\x18\x01 \x01(\x05R\nstatusCode\x12"\n\x0c\x65rrorMessage\x18\x02 \x01(\tR\x0c\x65rrorMessage\x12\x14\n\x05value\x18\x03 \x01(\x03R\x05value"\xb4\x04\n\x15StatefulProcessorCall\x12h\n\x0esetHandleState\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetHandleStateH\x00R\x0esetHandleState\x12h\n\rgetValueState\x18\x02 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\rgetValueState\x12\x66\n\x0cgetListState\x18\x03 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0cgetListState\x12\x64\n\x0bgetMapState\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0bgetMapState\x12o\n\x0etimerStateCall\x18\x05 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.TimerStateCallCommandH\x00R\x0etimerStateCallB\x08\n\x06method"\xd5\x02\n\x14StateVariableRequest\x12h\n\x0evalueStateCall\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.ValueStateCallH\x00R\x0evalueStateCall\x12\x65\n\rlistStateCall\x18\x02 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.ListStateCallH\x00R\rlistStateCall\x12\x62\n\x0cmapStateCall\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.MapStateCallH\x00R\x0cmapStateCallB\x08\n\x06method"\x83\x02\n\x1aImplicitGroupingKeyRequest\x12h\n\x0esetImplicitKey\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetImplicitKeyH\x00R\x0esetImplicitKey\x12q\n\x11removeImplicitKey\x18\x02 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.RemoveImplicitKeyH\x00R\x11removeImplicitKeyB\x08\n\x06method"\x81\x02\n\x0cTimerRequest\x12q\n\x11timerValueRequest\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.TimerValueRequestH\x00R\x11timerValueRequest\x12t\n\x12\x65xpiryTimerRequest\x18\x02 \x01(\x0b\x32\x42.org.apache.spark.sql.execution.streaming.state.ExpiryTimerRequestH\x00R\x12\x65xpiryTimerRequestB\x08\n\x06method"\xf6\x01\n\x11TimerValueRequest\x12s\n\x12getProcessingTimer\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.GetProcessingTimeH\x00R\x12getProcessingTimer\x12\x62\n\x0cgetWatermark\x18\x02 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.GetWatermarkH\x00R\x0cgetWatermarkB\x08\n\x06method"B\n\x12\x45xpiryTimerRequest\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs"\x13\n\x11GetProcessingTime"\x0e\n\x0cGetWatermark"\xc7\x01\n\x10StateCallCommand\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12\x16\n\x06schema\x18\x02 \x01(\tR\x06schema\x12\x30\n\x13mapStateValueSchema\x18\x03 \x01(\tR\x13mapStateValueSchema\x12K\n\x03ttl\x18\x04 \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.TTLConfigR\x03ttl"\xa7\x02\n\x15TimerStateCallCommand\x12[\n\x08register\x18\x01 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.RegisterTimerH\x00R\x08register\x12U\n\x06\x64\x65lete\x18\x02 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.DeleteTimerH\x00R\x06\x64\x65lete\x12P\n\x04list\x18\x03 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.ListTimersH\x00R\x04listB\x08\n\x06method"\x92\x03\n\x0eValueStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12G\n\x03get\x18\x03 \x01(\x0b\x32\x33.org.apache.spark.sql.execution.streaming.state.GetH\x00R\x03get\x12n\n\x10valueStateUpdate\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.ValueStateUpdateH\x00R\x10valueStateUpdate\x12M\n\x05\x63lear\x18\x05 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method"\xdf\x04\n\rListStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12\x62\n\x0clistStateGet\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStateGetH\x00R\x0clistStateGet\x12\x62\n\x0clistStatePut\x18\x04 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStatePutH\x00R\x0clistStatePut\x12_\n\x0b\x61ppendValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.AppendValueH\x00R\x0b\x61ppendValue\x12\\\n\nappendList\x18\x06 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.AppendListH\x00R\nappendList\x12M\n\x05\x63lear\x18\x07 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method"\xc2\x06\n\x0cMapStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12V\n\x08getValue\x18\x03 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.GetValueH\x00R\x08getValue\x12_\n\x0b\x63ontainsKey\x18\x04 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.ContainsKeyH\x00R\x0b\x63ontainsKey\x12_\n\x0bupdateValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.UpdateValueH\x00R\x0bupdateValue\x12V\n\x08iterator\x18\x06 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.IteratorH\x00R\x08iterator\x12J\n\x04keys\x18\x07 \x01(\x0b\x32\x34.org.apache.spark.sql.execution.streaming.state.KeysH\x00R\x04keys\x12P\n\x06values\x18\x08 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ValuesH\x00R\x06values\x12Y\n\tremoveKey\x18\t \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.RemoveKeyH\x00R\tremoveKey\x12M\n\x05\x63lear\x18\n \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method""\n\x0eSetImplicitKey\x12\x10\n\x03key\x18\x01 \x01(\x0cR\x03key"\x13\n\x11RemoveImplicitKey"\x08\n\x06\x45xists"\x05\n\x03Get"=\n\rRegisterTimer\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs";\n\x0b\x44\x65leteTimer\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs",\n\nListTimers\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"(\n\x10ValueStateUpdate\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value"\x07\n\x05\x43lear".\n\x0cListStateGet\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"\x0e\n\x0cListStatePut"#\n\x0b\x41ppendValue\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value"\x0c\n\nAppendList"$\n\x08GetValue\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"\'\n\x0b\x43ontainsKey\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"=\n\x0bUpdateValue\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value"*\n\x08Iterator\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"&\n\x04Keys\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"(\n\x06Values\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"%\n\tRemoveKey\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"c\n\x0eSetHandleState\x12Q\n\x05state\x18\x01 \x01(\x0e\x32;.org.apache.spark.sql.execution.streaming.state.HandleStateR\x05state"+\n\tTTLConfig\x12\x1e\n\ndurationMs\x18\x01 \x01(\x05R\ndurationMs*`\n\x0bHandleState\x12\x0b\n\x07\x43REATED\x10\x00\x12\x0f\n\x0bINITIALIZED\x10\x01\x12\x12\n\x0e\x44\x41TA_PROCESSED\x10\x02\x12\x13\n\x0fTIMER_PROCESSED\x10\x03\x12\n\n\x06\x43LOSED\x10\x04\x62\x06proto3' ) -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages( - DESCRIPTOR, "protobuf.org.apache.spark.sql.execution.streaming.StateMessage_pb2", globals() + DESCRIPTOR, "org.apache.spark.sql.execution.streaming.StateMessage_pb2", _globals ) -if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _HANDLESTATE._serialized_start = 5898 - _HANDLESTATE._serialized_end = 5994 - _STATEREQUEST._serialized_start = 121 - _STATEREQUEST._serialized_end = 665 - _STATERESPONSE._serialized_start = 667 - _STATERESPONSE._serialized_end = 772 - _STATERESPONSEWITHLONGTYPEVAL._serialized_start = 774 - _STATERESPONSEWITHLONGTYPEVAL._serialized_end = 894 - _STATEFULPROCESSORCALL._serialized_start = 897 - _STATEFULPROCESSORCALL._serialized_end = 1461 - _STATEVARIABLEREQUEST._serialized_start = 1464 - _STATEVARIABLEREQUEST._serialized_end = 1805 - _IMPLICITGROUPINGKEYREQUEST._serialized_start = 1808 - _IMPLICITGROUPINGKEYREQUEST._serialized_end = 2067 - _TIMERREQUEST._serialized_start = 2070 - _TIMERREQUEST._serialized_end = 2327 - _TIMERVALUEREQUEST._serialized_start = 2330 - _TIMERVALUEREQUEST._serialized_end = 2576 - _EXPIRYTIMERREQUEST._serialized_start = 2578 - _EXPIRYTIMERREQUEST._serialized_end = 2644 - _GETPROCESSINGTIME._serialized_start = 2646 - _GETPROCESSINGTIME._serialized_end = 2665 - _GETWATERMARK._serialized_start = 2667 - _GETWATERMARK._serialized_end = 2681 - _STATECALLCOMMAND._serialized_start = 2684 - _STATECALLCOMMAND._serialized_end = 2883 - _TIMERSTATECALLCOMMAND._serialized_start = 2886 - _TIMERSTATECALLCOMMAND._serialized_end = 3181 - _VALUESTATECALL._serialized_start = 3184 - _VALUESTATECALL._serialized_end = 3586 - _LISTSTATECALL._serialized_start = 3589 - _LISTSTATECALL._serialized_end = 4196 - _MAPSTATECALL._serialized_start = 4199 - _MAPSTATECALL._serialized_end = 5033 - _SETIMPLICITKEY._serialized_start = 5035 - _SETIMPLICITKEY._serialized_end = 5069 - _REMOVEIMPLICITKEY._serialized_start = 5071 - _REMOVEIMPLICITKEY._serialized_end = 5090 - _EXISTS._serialized_start = 5092 - _EXISTS._serialized_end = 5100 - _GET._serialized_start = 5102 - _GET._serialized_end = 5107 - _REGISTERTIMER._serialized_start = 5109 - _REGISTERTIMER._serialized_end = 5170 - _DELETETIMER._serialized_start = 5172 - _DELETETIMER._serialized_end = 5231 - _LISTTIMERS._serialized_start = 5233 - _LISTTIMERS._serialized_end = 5277 - _VALUESTATEUPDATE._serialized_start = 5279 - _VALUESTATEUPDATE._serialized_end = 5319 - _CLEAR._serialized_start = 5321 - _CLEAR._serialized_end = 5328 - _LISTSTATEGET._serialized_start = 5330 - _LISTSTATEGET._serialized_end = 5376 - _LISTSTATEPUT._serialized_start = 5378 - _LISTSTATEPUT._serialized_end = 5392 - _APPENDVALUE._serialized_start = 5394 - _APPENDVALUE._serialized_end = 5429 - _APPENDLIST._serialized_start = 5431 - _APPENDLIST._serialized_end = 5443 - _GETVALUE._serialized_start = 5445 - _GETVALUE._serialized_end = 5481 - _CONTAINSKEY._serialized_start = 5483 - _CONTAINSKEY._serialized_end = 5522 - _UPDATEVALUE._serialized_start = 5524 - _UPDATEVALUE._serialized_end = 5585 - _ITERATOR._serialized_start = 5587 - _ITERATOR._serialized_end = 5629 - _KEYS._serialized_start = 5631 - _KEYS._serialized_end = 5669 - _VALUES._serialized_start = 5671 - _VALUES._serialized_end = 5711 - _REMOVEKEY._serialized_start = 5713 - _REMOVEKEY._serialized_end = 5750 - _SETHANDLESTATE._serialized_start = 5752 - _SETHANDLESTATE._serialized_end = 5851 - _TTLCONFIG._serialized_start = 5853 - _TTLCONFIG._serialized_end = 5896 +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals["_HANDLESTATE"]._serialized_start = 5889 + _globals["_HANDLESTATE"]._serialized_end = 5985 + _globals["_STATEREQUEST"]._serialized_start = 112 + _globals["_STATEREQUEST"]._serialized_end = 656 + _globals["_STATERESPONSE"]._serialized_start = 658 + _globals["_STATERESPONSE"]._serialized_end = 763 + _globals["_STATERESPONSEWITHLONGTYPEVAL"]._serialized_start = 765 + _globals["_STATERESPONSEWITHLONGTYPEVAL"]._serialized_end = 885 + _globals["_STATEFULPROCESSORCALL"]._serialized_start = 888 + _globals["_STATEFULPROCESSORCALL"]._serialized_end = 1452 + _globals["_STATEVARIABLEREQUEST"]._serialized_start = 1455 + _globals["_STATEVARIABLEREQUEST"]._serialized_end = 1796 + _globals["_IMPLICITGROUPINGKEYREQUEST"]._serialized_start = 1799 + _globals["_IMPLICITGROUPINGKEYREQUEST"]._serialized_end = 2058 + _globals["_TIMERREQUEST"]._serialized_start = 2061 + _globals["_TIMERREQUEST"]._serialized_end = 2318 + _globals["_TIMERVALUEREQUEST"]._serialized_start = 2321 + _globals["_TIMERVALUEREQUEST"]._serialized_end = 2567 + _globals["_EXPIRYTIMERREQUEST"]._serialized_start = 2569 + _globals["_EXPIRYTIMERREQUEST"]._serialized_end = 2635 + _globals["_GETPROCESSINGTIME"]._serialized_start = 2637 + _globals["_GETPROCESSINGTIME"]._serialized_end = 2656 + _globals["_GETWATERMARK"]._serialized_start = 2658 + _globals["_GETWATERMARK"]._serialized_end = 2672 + _globals["_STATECALLCOMMAND"]._serialized_start = 2675 + _globals["_STATECALLCOMMAND"]._serialized_end = 2874 + _globals["_TIMERSTATECALLCOMMAND"]._serialized_start = 2877 + _globals["_TIMERSTATECALLCOMMAND"]._serialized_end = 3172 + _globals["_VALUESTATECALL"]._serialized_start = 3175 + _globals["_VALUESTATECALL"]._serialized_end = 3577 + _globals["_LISTSTATECALL"]._serialized_start = 3580 + _globals["_LISTSTATECALL"]._serialized_end = 4187 + _globals["_MAPSTATECALL"]._serialized_start = 4190 + _globals["_MAPSTATECALL"]._serialized_end = 5024 + _globals["_SETIMPLICITKEY"]._serialized_start = 5026 + _globals["_SETIMPLICITKEY"]._serialized_end = 5060 + _globals["_REMOVEIMPLICITKEY"]._serialized_start = 5062 + _globals["_REMOVEIMPLICITKEY"]._serialized_end = 5081 + _globals["_EXISTS"]._serialized_start = 5083 + _globals["_EXISTS"]._serialized_end = 5091 + _globals["_GET"]._serialized_start = 5093 + _globals["_GET"]._serialized_end = 5098 + _globals["_REGISTERTIMER"]._serialized_start = 5100 + _globals["_REGISTERTIMER"]._serialized_end = 5161 + _globals["_DELETETIMER"]._serialized_start = 5163 + _globals["_DELETETIMER"]._serialized_end = 5222 + _globals["_LISTTIMERS"]._serialized_start = 5224 + _globals["_LISTTIMERS"]._serialized_end = 5268 + _globals["_VALUESTATEUPDATE"]._serialized_start = 5270 + _globals["_VALUESTATEUPDATE"]._serialized_end = 5310 + _globals["_CLEAR"]._serialized_start = 5312 + _globals["_CLEAR"]._serialized_end = 5319 + _globals["_LISTSTATEGET"]._serialized_start = 5321 + _globals["_LISTSTATEGET"]._serialized_end = 5367 + _globals["_LISTSTATEPUT"]._serialized_start = 5369 + _globals["_LISTSTATEPUT"]._serialized_end = 5383 + _globals["_APPENDVALUE"]._serialized_start = 5385 + _globals["_APPENDVALUE"]._serialized_end = 5420 + _globals["_APPENDLIST"]._serialized_start = 5422 + _globals["_APPENDLIST"]._serialized_end = 5434 + _globals["_GETVALUE"]._serialized_start = 5436 + _globals["_GETVALUE"]._serialized_end = 5472 + _globals["_CONTAINSKEY"]._serialized_start = 5474 + _globals["_CONTAINSKEY"]._serialized_end = 5513 + _globals["_UPDATEVALUE"]._serialized_start = 5515 + _globals["_UPDATEVALUE"]._serialized_end = 5576 + _globals["_ITERATOR"]._serialized_start = 5578 + _globals["_ITERATOR"]._serialized_end = 5620 + _globals["_KEYS"]._serialized_start = 5622 + _globals["_KEYS"]._serialized_end = 5660 + _globals["_VALUES"]._serialized_start = 5662 + _globals["_VALUES"]._serialized_end = 5702 + _globals["_REMOVEKEY"]._serialized_start = 5704 + _globals["_REMOVEKEY"]._serialized_end = 5741 + _globals["_SETHANDLESTATE"]._serialized_start = 5743 + _globals["_SETHANDLESTATE"]._serialized_end = 5842 + _globals["_TTLCONFIG"]._serialized_start = 5844 + _globals["_TTLCONFIG"]._serialized_end = 5887 # @@protoc_insertion_point(module_scope) diff --git a/sql/core/src/main/buf.gen.yaml b/sql/core/src/main/buf.gen.yaml index d07abcca6ecc..de1c1bb564e1 100644 --- a/sql/core/src/main/buf.gen.yaml +++ b/sql/core/src/main/buf.gen.yaml @@ -17,7 +17,7 @@ version: v1 plugins: # Building the Python build and building the mypy interfaces. - - plugin: buf.build/protocolbuffers/python:v21.7 + - plugin: buf.build/protocolbuffers/python:v27.3 out: gen/proto/python - name: mypy out: gen/proto/python From 0f2e944a036fb0cacd5d21d0a6ed6dca0029d6c6 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 5 Nov 2024 21:54:59 +0800 Subject: [PATCH 03/14] support path --- dev/stream-state-gen-protos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/stream-state-gen-protos.sh b/dev/stream-state-gen-protos.sh index 48058ee6cf9a..b421e9751f32 100755 --- a/dev/stream-state-gen-protos.sh +++ b/dev/stream-state-gen-protos.sh @@ -18,9 +18,9 @@ # set -ex -if [[ $# -gt 0 ]]; then +if [[ $# -gt 1 ]]; then echo "Illegal number of parameters." - echo "Usage: ./dev/stream-state-gen-protos.sh" + echo "Usage: ./dev/stream-state-gen-protos.sh [path]" exit -1 fi From 31c950884d7120d4f46abed13c2885a5348a49e8 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 5 Nov 2024 22:11:34 +0800 Subject: [PATCH 04/14] add check --- .github/workflows/build_and_test.yml | 6 +- dev/streaming-state-check-protos.py | 84 +++++++++++++++++++ ...rotos.sh => streaming-state-gen-protos.sh} | 0 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100755 dev/streaming-state-check-protos.py rename dev/{stream-state-gen-protos.sh => streaming-state-gen-protos.sh} (100%) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index e51c896cc3f9..fd4f5da26669 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -617,7 +617,11 @@ jobs: python3.11 -m pip install 'black==23.9.1' 'protobuf==5.28.3' 'mypy==1.8.0' 'mypy-protobuf==3.3.0' python3.11 -m pip list - name: Python CodeGen check - run: ./dev/connect-check-protos.py + run: | + ./dev/connect-check-protos.py + if [ -f ./dev/streaming-state-check-protos.py ]; then + ./dev/streaming-state-check-protos.py + fi # Static analysis lint: diff --git a/dev/streaming-state-check-protos.py b/dev/streaming-state-check-protos.py new file mode 100755 index 000000000000..1eea3c953425 --- /dev/null +++ b/dev/streaming-state-check-protos.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Utility for checking whether generated codes in PySpark are out of sync. +# usage: ./dev/connect-check-protos.py + +import os +import sys +import filecmp +import tempfile +import subprocess + +# Location of your Spark git development area +SPARK_HOME = os.environ.get("SPARK_HOME", os.getcwd()) + + +def fail(msg): + print(msg) + sys.exit(-1) + + +def run_cmd(cmd): + print(f"RUN: {cmd}") + if isinstance(cmd, list): + return subprocess.check_output(cmd).decode("utf-8") + else: + return subprocess.check_output(cmd.split(" ")).decode("utf-8") + + +def check_streaming_protos(): + print("Start checking the generated codes in pyspark-streaming.") + with tempfile.TemporaryDirectory(prefix="check_streaming_protos") as tmp: + run_cmd(f"{SPARK_HOME}/dev/streaming-state-gen-protos.sh {tmp}") + result = filecmp.dircmp( + f"{SPARK_HOME}/python/pyspark/sql/streaming/proto/", + tmp, + ignore=["__init__.py", "__pycache__"], + ) + success = True + + if len(result.left_only) > 0: + print(f"Unexpected files: {result.left_only}") + success = False + + if len(result.right_only) > 0: + print(f"Missing files: {result.right_only}") + success = False + + if len(result.funny_files) > 0: + print(f"Incomparable files: {result.funny_files}") + success = False + + if len(result.diff_files) > 0: + print(f"Different files: {result.diff_files}") + success = False + + if success: + print("Finish checking the generated codes in pyspark-streaming: SUCCESS") + else: + fail( + "Generated files for pyspark-connect are out of sync! " + "If you have touched files under sql/core/src/main/protobuf/, " + "please run ./dev/streaming-state-gen-protos.sh. " + "If you haven't touched any file above, please rebase your PR against main branch." + ) + + +check_streaming_protos() diff --git a/dev/stream-state-gen-protos.sh b/dev/streaming-state-gen-protos.sh similarity index 100% rename from dev/stream-state-gen-protos.sh rename to dev/streaming-state-gen-protos.sh From 3d1a724c6fc98214dd1811901a6587bcf318c41d Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 5 Nov 2024 22:35:51 +0800 Subject: [PATCH 05/14] fix --- dev/streaming-state-check-protos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/streaming-state-check-protos.py b/dev/streaming-state-check-protos.py index 1eea3c953425..2b83c4ab0af6 100755 --- a/dev/streaming-state-check-protos.py +++ b/dev/streaming-state-check-protos.py @@ -18,7 +18,7 @@ # # Utility for checking whether generated codes in PySpark are out of sync. -# usage: ./dev/connect-check-protos.py +# usage: ./dev/streaming-state-gen-protos-check.py import os import sys @@ -74,7 +74,7 @@ def check_streaming_protos(): print("Finish checking the generated codes in pyspark-streaming: SUCCESS") else: fail( - "Generated files for pyspark-connect are out of sync! " + "Generated files for pyspark-streaming are out of sync! " "If you have touched files under sql/core/src/main/protobuf/, " "please run ./dev/streaming-state-gen-protos.sh. " "If you haven't touched any file above, please rebase your PR against main branch." From 6e17d077043e923cdc87362b7cc99fee910ed4a1 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 5 Nov 2024 23:46:29 +0800 Subject: [PATCH 06/14] upgrade to v28.3 --- python/pyspark/sql/streaming/proto/StateMessage_pb2.py | 4 ++-- sql/core/src/main/buf.gen.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py index fd175cb3a48a..17eeea530208 100644 --- a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py +++ b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py @@ -18,7 +18,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: org/apache/spark/sql/execution/streaming/StateMessage.proto -# Protobuf Python Version: 5.27.3 +# Protobuf Python Version: 5.28.3 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -29,7 +29,7 @@ _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, - 27, + 28, 3, "", "org/apache/spark/sql/execution/streaming/StateMessage.proto", diff --git a/sql/core/src/main/buf.gen.yaml b/sql/core/src/main/buf.gen.yaml index de1c1bb564e1..94da50c2c41c 100644 --- a/sql/core/src/main/buf.gen.yaml +++ b/sql/core/src/main/buf.gen.yaml @@ -17,7 +17,7 @@ version: v1 plugins: # Building the Python build and building the mypy interfaces. - - plugin: buf.build/protocolbuffers/python:v27.3 + - plugin: buf.build/protocolbuffers/python:v28.3 out: gen/proto/python - name: mypy out: gen/proto/python From 9ae8f085dae041ba5fcb6356f29c181b3e914d82 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 11 Nov 2024 11:00:33 +0800 Subject: [PATCH 07/14] re gen --- .../sql/streaming/proto/StateMessage_pb2.py | 173 +- .../sql/streaming/proto/StateMessage_pb2.pyi | 1552 ++++++++++++----- 2 files changed, 1206 insertions(+), 519 deletions(-) diff --git a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py index 46bed10c4558..a521e2968007 100644 --- a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py +++ b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py @@ -17,8 +17,8 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE -# source: StateMessage.proto -# Protobuf Python Version: 5.27.3 +# source: org/apache/spark/sql/execution/streaming/StateMessage.proto +# Protobuf Python Version: 5.28.3 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -27,7 +27,12 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, 5, 27, 3, "", "StateMessage.proto" + _runtime_version.Domain.PUBLIC, + 5, + 28, + 3, + "", + "org/apache/spark/sql/execution/streaming/StateMessage.proto", ) # @@protoc_insertion_point(imports) @@ -35,90 +40,92 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x12StateMessage.proto\x12.org.apache.spark.sql.execution.streaming.state"\xbf\x03\n\x0cStateRequest\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x66\n\x15statefulProcessorCall\x18\x02 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.StatefulProcessorCallH\x00\x12\x64\n\x14stateVariableRequest\x18\x03 \x01(\x0b\x32\x44.org.apache.spark.sql.execution.streaming.state.StateVariableRequestH\x00\x12p\n\x1aimplicitGroupingKeyRequest\x18\x04 \x01(\x0b\x32J.org.apache.spark.sql.execution.streaming.state.ImplicitGroupingKeyRequestH\x00\x12T\n\x0ctimerRequest\x18\x05 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.TimerRequestH\x00\x42\x08\n\x06method"H\n\rStateResponse\x12\x12\n\nstatusCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\x0c"W\n\x1cStateResponseWithLongTypeVal\x12\x12\n\nstatusCode\x18\x01 \x01(\x05\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\x03"\xc6\x04\n\x15StatefulProcessorCall\x12X\n\x0esetHandleState\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetHandleStateH\x00\x12Y\n\rgetValueState\x18\x02 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00\x12X\n\x0cgetListState\x18\x03 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00\x12W\n\x0bgetMapState\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00\x12_\n\x0etimerStateCall\x18\x05 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.TimerStateCallCommandH\x00\x12Z\n\x0e\x64\x65leteIfExists\x18\x06 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00\x42\x08\n\x06method"\xa8\x02\n\x14StateVariableRequest\x12X\n\x0evalueStateCall\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.ValueStateCallH\x00\x12V\n\rlistStateCall\x18\x02 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.ListStateCallH\x00\x12T\n\x0cmapStateCall\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.MapStateCallH\x00\x42\x08\n\x06method"\xe0\x01\n\x1aImplicitGroupingKeyRequest\x12X\n\x0esetImplicitKey\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetImplicitKeyH\x00\x12^\n\x11removeImplicitKey\x18\x02 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.RemoveImplicitKeyH\x00\x42\x08\n\x06method"\xda\x01\n\x0cTimerRequest\x12^\n\x11timerValueRequest\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.TimerValueRequestH\x00\x12`\n\x12\x65xpiryTimerRequest\x18\x02 \x01(\x0b\x32\x42.org.apache.spark.sql.execution.streaming.state.ExpiryTimerRequestH\x00\x42\x08\n\x06method"\xd4\x01\n\x11TimerValueRequest\x12_\n\x12getProcessingTimer\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.GetProcessingTimeH\x00\x12T\n\x0cgetWatermark\x18\x02 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.GetWatermarkH\x00\x42\x08\n\x06method"/\n\x12\x45xpiryTimerRequest\x12\x19\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03"\x13\n\x11GetProcessingTime"\x0e\n\x0cGetWatermark"\x9a\x01\n\x10StateCallCommand\x12\x11\n\tstateName\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\x1b\n\x13mapStateValueSchema\x18\x03 \x01(\t\x12\x46\n\x03ttl\x18\x04 \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.TTLConfig"\x8f\x02\n\x15TimerStateCallCommand\x12Q\n\x08register\x18\x01 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.RegisterTimerH\x00\x12M\n\x06\x64\x65lete\x18\x02 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.DeleteTimerH\x00\x12J\n\x04list\x18\x03 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.ListTimersH\x00\x42\x08\n\x06method"\xe1\x02\n\x0eValueStateCall\x12\x11\n\tstateName\x18\x01 \x01(\t\x12H\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00\x12\x42\n\x03get\x18\x03 \x01(\x0b\x32\x33.org.apache.spark.sql.execution.streaming.state.GetH\x00\x12\\\n\x10valueStateUpdate\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.ValueStateUpdateH\x00\x12\x46\n\x05\x63lear\x18\x05 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00\x42\x08\n\x06method"\x90\x04\n\rListStateCall\x12\x11\n\tstateName\x18\x01 \x01(\t\x12H\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00\x12T\n\x0clistStateGet\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStateGetH\x00\x12T\n\x0clistStatePut\x18\x04 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStatePutH\x00\x12R\n\x0b\x61ppendValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.AppendValueH\x00\x12P\n\nappendList\x18\x06 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.AppendListH\x00\x12\x46\n\x05\x63lear\x18\x07 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00\x42\x08\n\x06method"\xe1\x05\n\x0cMapStateCall\x12\x11\n\tstateName\x18\x01 \x01(\t\x12H\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00\x12L\n\x08getValue\x18\x03 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.GetValueH\x00\x12R\n\x0b\x63ontainsKey\x18\x04 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.ContainsKeyH\x00\x12R\n\x0bupdateValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.UpdateValueH\x00\x12L\n\x08iterator\x18\x06 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.IteratorH\x00\x12\x44\n\x04keys\x18\x07 \x01(\x0b\x32\x34.org.apache.spark.sql.execution.streaming.state.KeysH\x00\x12H\n\x06values\x18\x08 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ValuesH\x00\x12N\n\tremoveKey\x18\t \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.RemoveKeyH\x00\x12\x46\n\x05\x63lear\x18\n \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00\x42\x08\n\x06method"\x1d\n\x0eSetImplicitKey\x12\x0b\n\x03key\x18\x01 \x01(\x0c"\x13\n\x11RemoveImplicitKey"\x08\n\x06\x45xists"\x05\n\x03Get"*\n\rRegisterTimer\x12\x19\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03"(\n\x0b\x44\x65leteTimer\x12\x19\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03" \n\nListTimers\x12\x12\n\niteratorId\x18\x01 \x01(\t"!\n\x10ValueStateUpdate\x12\r\n\x05value\x18\x01 \x01(\x0c"\x07\n\x05\x43lear""\n\x0cListStateGet\x12\x12\n\niteratorId\x18\x01 \x01(\t"\x0e\n\x0cListStatePut"\x1c\n\x0b\x41ppendValue\x12\r\n\x05value\x18\x01 \x01(\x0c"\x0c\n\nAppendList"\x1b\n\x08GetValue\x12\x0f\n\x07userKey\x18\x01 \x01(\x0c"\x1e\n\x0b\x43ontainsKey\x12\x0f\n\x07userKey\x18\x01 \x01(\x0c"-\n\x0bUpdateValue\x12\x0f\n\x07userKey\x18\x01 \x01(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c"\x1e\n\x08Iterator\x12\x12\n\niteratorId\x18\x01 \x01(\t"\x1a\n\x04Keys\x12\x12\n\niteratorId\x18\x01 \x01(\t"\x1c\n\x06Values\x12\x12\n\niteratorId\x18\x01 \x01(\t"\x1c\n\tRemoveKey\x12\x0f\n\x07userKey\x18\x01 \x01(\x0c"\\\n\x0eSetHandleState\x12J\n\x05state\x18\x01 \x01(\x0e\x32;.org.apache.spark.sql.execution.streaming.state.HandleState"\x1f\n\tTTLConfig\x12\x12\n\ndurationMs\x18\x01 \x01(\x05*`\n\x0bHandleState\x12\x0b\n\x07\x43REATED\x10\x00\x12\x0f\n\x0bINITIALIZED\x10\x01\x12\x12\n\x0e\x44\x41TA_PROCESSED\x10\x02\x12\x13\n\x0fTIMER_PROCESSED\x10\x03\x12\n\n\x06\x43LOSED\x10\x04\x62\x06proto3' # noqa: E501 + b'\n;org/apache/spark/sql/execution/streaming/StateMessage.proto\x12.org.apache.spark.sql.execution.streaming.state"\xa0\x04\n\x0cStateRequest\x12\x18\n\x07version\x18\x01 \x01(\x05R\x07version\x12}\n\x15statefulProcessorCall\x18\x02 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.StatefulProcessorCallH\x00R\x15statefulProcessorCall\x12z\n\x14stateVariableRequest\x18\x03 \x01(\x0b\x32\x44.org.apache.spark.sql.execution.streaming.state.StateVariableRequestH\x00R\x14stateVariableRequest\x12\x8c\x01\n\x1aimplicitGroupingKeyRequest\x18\x04 \x01(\x0b\x32J.org.apache.spark.sql.execution.streaming.state.ImplicitGroupingKeyRequestH\x00R\x1aimplicitGroupingKeyRequest\x12\x62\n\x0ctimerRequest\x18\x05 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.TimerRequestH\x00R\x0ctimerRequestB\x08\n\x06method"i\n\rStateResponse\x12\x1e\n\nstatusCode\x18\x01 \x01(\x05R\nstatusCode\x12"\n\x0c\x65rrorMessage\x18\x02 \x01(\tR\x0c\x65rrorMessage\x12\x14\n\x05value\x18\x03 \x01(\x0cR\x05value"x\n\x1cStateResponseWithLongTypeVal\x12\x1e\n\nstatusCode\x18\x01 \x01(\x05R\nstatusCode\x12"\n\x0c\x65rrorMessage\x18\x02 \x01(\tR\x0c\x65rrorMessage\x12\x14\n\x05value\x18\x03 \x01(\x03R\x05value"\xa0\x05\n\x15StatefulProcessorCall\x12h\n\x0esetHandleState\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetHandleStateH\x00R\x0esetHandleState\x12h\n\rgetValueState\x18\x02 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\rgetValueState\x12\x66\n\x0cgetListState\x18\x03 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0cgetListState\x12\x64\n\x0bgetMapState\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0bgetMapState\x12o\n\x0etimerStateCall\x18\x05 \x01(\x0b\x32\x45.org.apache.spark.sql.execution.streaming.state.TimerStateCallCommandH\x00R\x0etimerStateCall\x12j\n\x0e\x64\x65leteIfExists\x18\x06 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.StateCallCommandH\x00R\x0e\x64\x65leteIfExistsB\x08\n\x06method"\xd5\x02\n\x14StateVariableRequest\x12h\n\x0evalueStateCall\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.ValueStateCallH\x00R\x0evalueStateCall\x12\x65\n\rlistStateCall\x18\x02 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.ListStateCallH\x00R\rlistStateCall\x12\x62\n\x0cmapStateCall\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.MapStateCallH\x00R\x0cmapStateCallB\x08\n\x06method"\x83\x02\n\x1aImplicitGroupingKeyRequest\x12h\n\x0esetImplicitKey\x18\x01 \x01(\x0b\x32>.org.apache.spark.sql.execution.streaming.state.SetImplicitKeyH\x00R\x0esetImplicitKey\x12q\n\x11removeImplicitKey\x18\x02 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.RemoveImplicitKeyH\x00R\x11removeImplicitKeyB\x08\n\x06method"\x81\x02\n\x0cTimerRequest\x12q\n\x11timerValueRequest\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.TimerValueRequestH\x00R\x11timerValueRequest\x12t\n\x12\x65xpiryTimerRequest\x18\x02 \x01(\x0b\x32\x42.org.apache.spark.sql.execution.streaming.state.ExpiryTimerRequestH\x00R\x12\x65xpiryTimerRequestB\x08\n\x06method"\xf6\x01\n\x11TimerValueRequest\x12s\n\x12getProcessingTimer\x18\x01 \x01(\x0b\x32\x41.org.apache.spark.sql.execution.streaming.state.GetProcessingTimeH\x00R\x12getProcessingTimer\x12\x62\n\x0cgetWatermark\x18\x02 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.GetWatermarkH\x00R\x0cgetWatermarkB\x08\n\x06method"B\n\x12\x45xpiryTimerRequest\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs"\x13\n\x11GetProcessingTime"\x0e\n\x0cGetWatermark"\xc7\x01\n\x10StateCallCommand\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12\x16\n\x06schema\x18\x02 \x01(\tR\x06schema\x12\x30\n\x13mapStateValueSchema\x18\x03 \x01(\tR\x13mapStateValueSchema\x12K\n\x03ttl\x18\x04 \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.TTLConfigR\x03ttl"\xa7\x02\n\x15TimerStateCallCommand\x12[\n\x08register\x18\x01 \x01(\x0b\x32=.org.apache.spark.sql.execution.streaming.state.RegisterTimerH\x00R\x08register\x12U\n\x06\x64\x65lete\x18\x02 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.DeleteTimerH\x00R\x06\x64\x65lete\x12P\n\x04list\x18\x03 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.ListTimersH\x00R\x04listB\x08\n\x06method"\x92\x03\n\x0eValueStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12G\n\x03get\x18\x03 \x01(\x0b\x32\x33.org.apache.spark.sql.execution.streaming.state.GetH\x00R\x03get\x12n\n\x10valueStateUpdate\x18\x04 \x01(\x0b\x32@.org.apache.spark.sql.execution.streaming.state.ValueStateUpdateH\x00R\x10valueStateUpdate\x12M\n\x05\x63lear\x18\x05 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method"\xdf\x04\n\rListStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12\x62\n\x0clistStateGet\x18\x03 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStateGetH\x00R\x0clistStateGet\x12\x62\n\x0clistStatePut\x18\x04 \x01(\x0b\x32<.org.apache.spark.sql.execution.streaming.state.ListStatePutH\x00R\x0clistStatePut\x12_\n\x0b\x61ppendValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.AppendValueH\x00R\x0b\x61ppendValue\x12\\\n\nappendList\x18\x06 \x01(\x0b\x32:.org.apache.spark.sql.execution.streaming.state.AppendListH\x00R\nappendList\x12M\n\x05\x63lear\x18\x07 \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method"\xc2\x06\n\x0cMapStateCall\x12\x1c\n\tstateName\x18\x01 \x01(\tR\tstateName\x12P\n\x06\x65xists\x18\x02 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ExistsH\x00R\x06\x65xists\x12V\n\x08getValue\x18\x03 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.GetValueH\x00R\x08getValue\x12_\n\x0b\x63ontainsKey\x18\x04 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.ContainsKeyH\x00R\x0b\x63ontainsKey\x12_\n\x0bupdateValue\x18\x05 \x01(\x0b\x32;.org.apache.spark.sql.execution.streaming.state.UpdateValueH\x00R\x0bupdateValue\x12V\n\x08iterator\x18\x06 \x01(\x0b\x32\x38.org.apache.spark.sql.execution.streaming.state.IteratorH\x00R\x08iterator\x12J\n\x04keys\x18\x07 \x01(\x0b\x32\x34.org.apache.spark.sql.execution.streaming.state.KeysH\x00R\x04keys\x12P\n\x06values\x18\x08 \x01(\x0b\x32\x36.org.apache.spark.sql.execution.streaming.state.ValuesH\x00R\x06values\x12Y\n\tremoveKey\x18\t \x01(\x0b\x32\x39.org.apache.spark.sql.execution.streaming.state.RemoveKeyH\x00R\tremoveKey\x12M\n\x05\x63lear\x18\n \x01(\x0b\x32\x35.org.apache.spark.sql.execution.streaming.state.ClearH\x00R\x05\x63learB\x08\n\x06method""\n\x0eSetImplicitKey\x12\x10\n\x03key\x18\x01 \x01(\x0cR\x03key"\x13\n\x11RemoveImplicitKey"\x08\n\x06\x45xists"\x05\n\x03Get"=\n\rRegisterTimer\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs";\n\x0b\x44\x65leteTimer\x12,\n\x11\x65xpiryTimestampMs\x18\x01 \x01(\x03R\x11\x65xpiryTimestampMs",\n\nListTimers\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"(\n\x10ValueStateUpdate\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value"\x07\n\x05\x43lear".\n\x0cListStateGet\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"\x0e\n\x0cListStatePut"#\n\x0b\x41ppendValue\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05value"\x0c\n\nAppendList"$\n\x08GetValue\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"\'\n\x0b\x43ontainsKey\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"=\n\x0bUpdateValue\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value"*\n\x08Iterator\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"&\n\x04Keys\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"(\n\x06Values\x12\x1e\n\niteratorId\x18\x01 \x01(\tR\niteratorId"%\n\tRemoveKey\x12\x18\n\x07userKey\x18\x01 \x01(\x0cR\x07userKey"c\n\x0eSetHandleState\x12Q\n\x05state\x18\x01 \x01(\x0e\x32;.org.apache.spark.sql.execution.streaming.state.HandleStateR\x05state"+\n\tTTLConfig\x12\x1e\n\ndurationMs\x18\x01 \x01(\x05R\ndurationMs*`\n\x0bHandleState\x12\x0b\n\x07\x43REATED\x10\x00\x12\x0f\n\x0bINITIALIZED\x10\x01\x12\x12\n\x0e\x44\x41TA_PROCESSED\x10\x02\x12\x13\n\x0fTIMER_PROCESSED\x10\x03\x12\n\n\x06\x43LOSED\x10\x04\x62\x06proto3' ) _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "StateMessage_pb2", _globals) +_builder.BuildTopDescriptorsAndMessages( + DESCRIPTOR, "org.apache.spark.sql.execution.streaming.StateMessage_pb2", _globals +) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None - _globals["_HANDLESTATE"]._serialized_start = 5058 - _globals["_HANDLESTATE"]._serialized_end = 5154 - _globals["_STATEREQUEST"]._serialized_start = 71 - _globals["_STATEREQUEST"]._serialized_end = 518 - _globals["_STATERESPONSE"]._serialized_start = 520 - _globals["_STATERESPONSE"]._serialized_end = 592 - _globals["_STATERESPONSEWITHLONGTYPEVAL"]._serialized_start = 594 - _globals["_STATERESPONSEWITHLONGTYPEVAL"]._serialized_end = 681 - _globals["_STATEFULPROCESSORCALL"]._serialized_start = 684 - _globals["_STATEFULPROCESSORCALL"]._serialized_end = 1266 - _globals["_STATEVARIABLEREQUEST"]._serialized_start = 1269 - _globals["_STATEVARIABLEREQUEST"]._serialized_end = 1565 - _globals["_IMPLICITGROUPINGKEYREQUEST"]._serialized_start = 1568 - _globals["_IMPLICITGROUPINGKEYREQUEST"]._serialized_end = 1792 - _globals["_TIMERREQUEST"]._serialized_start = 1795 - _globals["_TIMERREQUEST"]._serialized_end = 2013 - _globals["_TIMERVALUEREQUEST"]._serialized_start = 2016 - _globals["_TIMERVALUEREQUEST"]._serialized_end = 2228 - _globals["_EXPIRYTIMERREQUEST"]._serialized_start = 2230 - _globals["_EXPIRYTIMERREQUEST"]._serialized_end = 2277 - _globals["_GETPROCESSINGTIME"]._serialized_start = 2279 - _globals["_GETPROCESSINGTIME"]._serialized_end = 2298 - _globals["_GETWATERMARK"]._serialized_start = 2300 - _globals["_GETWATERMARK"]._serialized_end = 2314 - _globals["_STATECALLCOMMAND"]._serialized_start = 2317 - _globals["_STATECALLCOMMAND"]._serialized_end = 2471 - _globals["_TIMERSTATECALLCOMMAND"]._serialized_start = 2474 - _globals["_TIMERSTATECALLCOMMAND"]._serialized_end = 2745 - _globals["_VALUESTATECALL"]._serialized_start = 2748 - _globals["_VALUESTATECALL"]._serialized_end = 3101 - _globals["_LISTSTATECALL"]._serialized_start = 3104 - _globals["_LISTSTATECALL"]._serialized_end = 3632 - _globals["_MAPSTATECALL"]._serialized_start = 3635 - _globals["_MAPSTATECALL"]._serialized_end = 4372 - _globals["_SETIMPLICITKEY"]._serialized_start = 4374 - _globals["_SETIMPLICITKEY"]._serialized_end = 4403 - _globals["_REMOVEIMPLICITKEY"]._serialized_start = 4405 - _globals["_REMOVEIMPLICITKEY"]._serialized_end = 4424 - _globals["_EXISTS"]._serialized_start = 4426 - _globals["_EXISTS"]._serialized_end = 4434 - _globals["_GET"]._serialized_start = 4436 - _globals["_GET"]._serialized_end = 4441 - _globals["_REGISTERTIMER"]._serialized_start = 4443 - _globals["_REGISTERTIMER"]._serialized_end = 4485 - _globals["_DELETETIMER"]._serialized_start = 4487 - _globals["_DELETETIMER"]._serialized_end = 4527 - _globals["_LISTTIMERS"]._serialized_start = 4529 - _globals["_LISTTIMERS"]._serialized_end = 4561 - _globals["_VALUESTATEUPDATE"]._serialized_start = 4563 - _globals["_VALUESTATEUPDATE"]._serialized_end = 4596 - _globals["_CLEAR"]._serialized_start = 4598 - _globals["_CLEAR"]._serialized_end = 4605 - _globals["_LISTSTATEGET"]._serialized_start = 4607 - _globals["_LISTSTATEGET"]._serialized_end = 4641 - _globals["_LISTSTATEPUT"]._serialized_start = 4643 - _globals["_LISTSTATEPUT"]._serialized_end = 4657 - _globals["_APPENDVALUE"]._serialized_start = 4659 - _globals["_APPENDVALUE"]._serialized_end = 4687 - _globals["_APPENDLIST"]._serialized_start = 4689 - _globals["_APPENDLIST"]._serialized_end = 4701 - _globals["_GETVALUE"]._serialized_start = 4703 - _globals["_GETVALUE"]._serialized_end = 4730 - _globals["_CONTAINSKEY"]._serialized_start = 4732 - _globals["_CONTAINSKEY"]._serialized_end = 4762 - _globals["_UPDATEVALUE"]._serialized_start = 4764 - _globals["_UPDATEVALUE"]._serialized_end = 4809 - _globals["_ITERATOR"]._serialized_start = 4811 - _globals["_ITERATOR"]._serialized_end = 4841 - _globals["_KEYS"]._serialized_start = 4843 - _globals["_KEYS"]._serialized_end = 4869 - _globals["_VALUES"]._serialized_start = 4871 - _globals["_VALUES"]._serialized_end = 4899 - _globals["_REMOVEKEY"]._serialized_start = 4901 - _globals["_REMOVEKEY"]._serialized_end = 4929 - _globals["_SETHANDLESTATE"]._serialized_start = 4931 - _globals["_SETHANDLESTATE"]._serialized_end = 5023 - _globals["_TTLCONFIG"]._serialized_start = 5025 - _globals["_TTLCONFIG"]._serialized_end = 5056 + _globals["_HANDLESTATE"]._serialized_start = 5997 + _globals["_HANDLESTATE"]._serialized_end = 6093 + _globals["_STATEREQUEST"]._serialized_start = 112 + _globals["_STATEREQUEST"]._serialized_end = 656 + _globals["_STATERESPONSE"]._serialized_start = 658 + _globals["_STATERESPONSE"]._serialized_end = 763 + _globals["_STATERESPONSEWITHLONGTYPEVAL"]._serialized_start = 765 + _globals["_STATERESPONSEWITHLONGTYPEVAL"]._serialized_end = 885 + _globals["_STATEFULPROCESSORCALL"]._serialized_start = 888 + _globals["_STATEFULPROCESSORCALL"]._serialized_end = 1560 + _globals["_STATEVARIABLEREQUEST"]._serialized_start = 1563 + _globals["_STATEVARIABLEREQUEST"]._serialized_end = 1904 + _globals["_IMPLICITGROUPINGKEYREQUEST"]._serialized_start = 1907 + _globals["_IMPLICITGROUPINGKEYREQUEST"]._serialized_end = 2166 + _globals["_TIMERREQUEST"]._serialized_start = 2169 + _globals["_TIMERREQUEST"]._serialized_end = 2426 + _globals["_TIMERVALUEREQUEST"]._serialized_start = 2429 + _globals["_TIMERVALUEREQUEST"]._serialized_end = 2675 + _globals["_EXPIRYTIMERREQUEST"]._serialized_start = 2677 + _globals["_EXPIRYTIMERREQUEST"]._serialized_end = 2743 + _globals["_GETPROCESSINGTIME"]._serialized_start = 2745 + _globals["_GETPROCESSINGTIME"]._serialized_end = 2764 + _globals["_GETWATERMARK"]._serialized_start = 2766 + _globals["_GETWATERMARK"]._serialized_end = 2780 + _globals["_STATECALLCOMMAND"]._serialized_start = 2783 + _globals["_STATECALLCOMMAND"]._serialized_end = 2982 + _globals["_TIMERSTATECALLCOMMAND"]._serialized_start = 2985 + _globals["_TIMERSTATECALLCOMMAND"]._serialized_end = 3280 + _globals["_VALUESTATECALL"]._serialized_start = 3283 + _globals["_VALUESTATECALL"]._serialized_end = 3685 + _globals["_LISTSTATECALL"]._serialized_start = 3688 + _globals["_LISTSTATECALL"]._serialized_end = 4295 + _globals["_MAPSTATECALL"]._serialized_start = 4298 + _globals["_MAPSTATECALL"]._serialized_end = 5132 + _globals["_SETIMPLICITKEY"]._serialized_start = 5134 + _globals["_SETIMPLICITKEY"]._serialized_end = 5168 + _globals["_REMOVEIMPLICITKEY"]._serialized_start = 5170 + _globals["_REMOVEIMPLICITKEY"]._serialized_end = 5189 + _globals["_EXISTS"]._serialized_start = 5191 + _globals["_EXISTS"]._serialized_end = 5199 + _globals["_GET"]._serialized_start = 5201 + _globals["_GET"]._serialized_end = 5206 + _globals["_REGISTERTIMER"]._serialized_start = 5208 + _globals["_REGISTERTIMER"]._serialized_end = 5269 + _globals["_DELETETIMER"]._serialized_start = 5271 + _globals["_DELETETIMER"]._serialized_end = 5330 + _globals["_LISTTIMERS"]._serialized_start = 5332 + _globals["_LISTTIMERS"]._serialized_end = 5376 + _globals["_VALUESTATEUPDATE"]._serialized_start = 5378 + _globals["_VALUESTATEUPDATE"]._serialized_end = 5418 + _globals["_CLEAR"]._serialized_start = 5420 + _globals["_CLEAR"]._serialized_end = 5427 + _globals["_LISTSTATEGET"]._serialized_start = 5429 + _globals["_LISTSTATEGET"]._serialized_end = 5475 + _globals["_LISTSTATEPUT"]._serialized_start = 5477 + _globals["_LISTSTATEPUT"]._serialized_end = 5491 + _globals["_APPENDVALUE"]._serialized_start = 5493 + _globals["_APPENDVALUE"]._serialized_end = 5528 + _globals["_APPENDLIST"]._serialized_start = 5530 + _globals["_APPENDLIST"]._serialized_end = 5542 + _globals["_GETVALUE"]._serialized_start = 5544 + _globals["_GETVALUE"]._serialized_end = 5580 + _globals["_CONTAINSKEY"]._serialized_start = 5582 + _globals["_CONTAINSKEY"]._serialized_end = 5621 + _globals["_UPDATEVALUE"]._serialized_start = 5623 + _globals["_UPDATEVALUE"]._serialized_end = 5684 + _globals["_ITERATOR"]._serialized_start = 5686 + _globals["_ITERATOR"]._serialized_end = 5728 + _globals["_KEYS"]._serialized_start = 5730 + _globals["_KEYS"]._serialized_end = 5768 + _globals["_VALUES"]._serialized_start = 5770 + _globals["_VALUES"]._serialized_end = 5810 + _globals["_REMOVEKEY"]._serialized_start = 5812 + _globals["_REMOVEKEY"]._serialized_end = 5849 + _globals["_SETHANDLESTATE"]._serialized_start = 5851 + _globals["_SETHANDLESTATE"]._serialized_end = 5950 + _globals["_TTLCONFIG"]._serialized_start = 5952 + _globals["_TTLCONFIG"]._serialized_end = 5995 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi b/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi index bc5138f52281..52f66928294c 100644 --- a/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi +++ b/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi @@ -14,439 +14,1119 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ( - ClassVar as _ClassVar, - Mapping as _Mapping, - Optional as _Optional, - Union as _Union, -) - -DESCRIPTOR: _descriptor.FileDescriptor - -class HandleState(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = () - CREATED: _ClassVar[HandleState] - INITIALIZED: _ClassVar[HandleState] - DATA_PROCESSED: _ClassVar[HandleState] - TIMER_PROCESSED: _ClassVar[HandleState] - CLOSED: _ClassVar[HandleState] - -CREATED: HandleState -INITIALIZED: HandleState -DATA_PROCESSED: HandleState -TIMER_PROCESSED: HandleState -CLOSED: HandleState - -class StateRequest(_message.Message): - __slots__ = ( - "version", - "statefulProcessorCall", - "stateVariableRequest", - "implicitGroupingKeyRequest", - "timerRequest", - ) - VERSION_FIELD_NUMBER: _ClassVar[int] - STATEFULPROCESSORCALL_FIELD_NUMBER: _ClassVar[int] - STATEVARIABLEREQUEST_FIELD_NUMBER: _ClassVar[int] - IMPLICITGROUPINGKEYREQUEST_FIELD_NUMBER: _ClassVar[int] - TIMERREQUEST_FIELD_NUMBER: _ClassVar[int] - version: int - statefulProcessorCall: StatefulProcessorCall - stateVariableRequest: StateVariableRequest - implicitGroupingKeyRequest: ImplicitGroupingKeyRequest - timerRequest: TimerRequest - def __init__( - self, - version: _Optional[int] = ..., - statefulProcessorCall: _Optional[_Union[StatefulProcessorCall, _Mapping]] = ..., - stateVariableRequest: _Optional[_Union[StateVariableRequest, _Mapping]] = ..., - implicitGroupingKeyRequest: _Optional[_Union[ImplicitGroupingKeyRequest, _Mapping]] = ..., - timerRequest: _Optional[_Union[TimerRequest, _Mapping]] = ..., - ) -> None: ... - -class StateResponse(_message.Message): - __slots__ = ("statusCode", "errorMessage", "value") - STATUSCODE_FIELD_NUMBER: _ClassVar[int] - ERRORMESSAGE_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - statusCode: int - errorMessage: str - value: bytes - def __init__( - self, - statusCode: _Optional[int] = ..., - errorMessage: _Optional[str] = ..., - value: _Optional[bytes] = ..., - ) -> None: ... - -class StateResponseWithLongTypeVal(_message.Message): - __slots__ = ("statusCode", "errorMessage", "value") - STATUSCODE_FIELD_NUMBER: _ClassVar[int] - ERRORMESSAGE_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - statusCode: int - errorMessage: str - value: int - def __init__( - self, - statusCode: _Optional[int] = ..., - errorMessage: _Optional[str] = ..., - value: _Optional[int] = ..., - ) -> None: ... - -class StatefulProcessorCall(_message.Message): - __slots__ = ( - "setHandleState", - "getValueState", - "getListState", - "getMapState", - "timerStateCall", - "deleteIfExists", - ) - SETHANDLESTATE_FIELD_NUMBER: _ClassVar[int] - GETVALUESTATE_FIELD_NUMBER: _ClassVar[int] - GETLISTSTATE_FIELD_NUMBER: _ClassVar[int] - GETMAPSTATE_FIELD_NUMBER: _ClassVar[int] - TIMERSTATECALL_FIELD_NUMBER: _ClassVar[int] - DELETEIFEXISTS_FIELD_NUMBER: _ClassVar[int] - setHandleState: SetHandleState - getValueState: StateCallCommand - getListState: StateCallCommand - getMapState: StateCallCommand - timerStateCall: TimerStateCallCommand - deleteIfExists: StateCallCommand - def __init__( - self, - setHandleState: _Optional[_Union[SetHandleState, _Mapping]] = ..., - getValueState: _Optional[_Union[StateCallCommand, _Mapping]] = ..., - getListState: _Optional[_Union[StateCallCommand, _Mapping]] = ..., - getMapState: _Optional[_Union[StateCallCommand, _Mapping]] = ..., - timerStateCall: _Optional[_Union[TimerStateCallCommand, _Mapping]] = ..., - deleteIfExists: _Optional[_Union[StateCallCommand, _Mapping]] = ..., - ) -> None: ... - -class StateVariableRequest(_message.Message): - __slots__ = ("valueStateCall", "listStateCall", "mapStateCall") - VALUESTATECALL_FIELD_NUMBER: _ClassVar[int] - LISTSTATECALL_FIELD_NUMBER: _ClassVar[int] - MAPSTATECALL_FIELD_NUMBER: _ClassVar[int] - valueStateCall: ValueStateCall - listStateCall: ListStateCall - mapStateCall: MapStateCall - def __init__( - self, - valueStateCall: _Optional[_Union[ValueStateCall, _Mapping]] = ..., - listStateCall: _Optional[_Union[ListStateCall, _Mapping]] = ..., - mapStateCall: _Optional[_Union[MapStateCall, _Mapping]] = ..., - ) -> None: ... - -class ImplicitGroupingKeyRequest(_message.Message): - __slots__ = ("setImplicitKey", "removeImplicitKey") - SETIMPLICITKEY_FIELD_NUMBER: _ClassVar[int] - REMOVEIMPLICITKEY_FIELD_NUMBER: _ClassVar[int] - setImplicitKey: SetImplicitKey - removeImplicitKey: RemoveImplicitKey - def __init__( - self, - setImplicitKey: _Optional[_Union[SetImplicitKey, _Mapping]] = ..., - removeImplicitKey: _Optional[_Union[RemoveImplicitKey, _Mapping]] = ..., - ) -> None: ... - -class TimerRequest(_message.Message): - __slots__ = ("timerValueRequest", "expiryTimerRequest") - TIMERVALUEREQUEST_FIELD_NUMBER: _ClassVar[int] - EXPIRYTIMERREQUEST_FIELD_NUMBER: _ClassVar[int] - timerValueRequest: TimerValueRequest - expiryTimerRequest: ExpiryTimerRequest - def __init__( - self, - timerValueRequest: _Optional[_Union[TimerValueRequest, _Mapping]] = ..., - expiryTimerRequest: _Optional[_Union[ExpiryTimerRequest, _Mapping]] = ..., - ) -> None: ... - -class TimerValueRequest(_message.Message): - __slots__ = ("getProcessingTimer", "getWatermark") - GETPROCESSINGTIMER_FIELD_NUMBER: _ClassVar[int] - GETWATERMARK_FIELD_NUMBER: _ClassVar[int] - getProcessingTimer: GetProcessingTime - getWatermark: GetWatermark - def __init__( - self, - getProcessingTimer: _Optional[_Union[GetProcessingTime, _Mapping]] = ..., - getWatermark: _Optional[_Union[GetWatermark, _Mapping]] = ..., - ) -> None: ... - -class ExpiryTimerRequest(_message.Message): - __slots__ = ("expiryTimestampMs",) - EXPIRYTIMESTAMPMS_FIELD_NUMBER: _ClassVar[int] - expiryTimestampMs: int - def __init__(self, expiryTimestampMs: _Optional[int] = ...) -> None: ... - -class GetProcessingTime(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class GetWatermark(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class StateCallCommand(_message.Message): - __slots__ = ("stateName", "schema", "mapStateValueSchema", "ttl") - STATENAME_FIELD_NUMBER: _ClassVar[int] - SCHEMA_FIELD_NUMBER: _ClassVar[int] - MAPSTATEVALUESCHEMA_FIELD_NUMBER: _ClassVar[int] - TTL_FIELD_NUMBER: _ClassVar[int] - stateName: str - schema: str - mapStateValueSchema: str - ttl: TTLConfig - def __init__( - self, - stateName: _Optional[str] = ..., - schema: _Optional[str] = ..., - mapStateValueSchema: _Optional[str] = ..., - ttl: _Optional[_Union[TTLConfig, _Mapping]] = ..., - ) -> None: ... - -class TimerStateCallCommand(_message.Message): - __slots__ = ("register", "delete", "list") - REGISTER_FIELD_NUMBER: _ClassVar[int] - DELETE_FIELD_NUMBER: _ClassVar[int] - LIST_FIELD_NUMBER: _ClassVar[int] - register: RegisterTimer - delete: DeleteTimer - list: ListTimers - def __init__( - self, - register: _Optional[_Union[RegisterTimer, _Mapping]] = ..., - delete: _Optional[_Union[DeleteTimer, _Mapping]] = ..., - list: _Optional[_Union[ListTimers, _Mapping]] = ..., - ) -> None: ... - -class ValueStateCall(_message.Message): - __slots__ = ("stateName", "exists", "get", "valueStateUpdate", "clear") - STATENAME_FIELD_NUMBER: _ClassVar[int] - EXISTS_FIELD_NUMBER: _ClassVar[int] - GET_FIELD_NUMBER: _ClassVar[int] - VALUESTATEUPDATE_FIELD_NUMBER: _ClassVar[int] - CLEAR_FIELD_NUMBER: _ClassVar[int] - stateName: str - exists: Exists - get: Get - valueStateUpdate: ValueStateUpdate - clear: Clear - def __init__( - self, - stateName: _Optional[str] = ..., - exists: _Optional[_Union[Exists, _Mapping]] = ..., - get: _Optional[_Union[Get, _Mapping]] = ..., - valueStateUpdate: _Optional[_Union[ValueStateUpdate, _Mapping]] = ..., - clear: _Optional[_Union[Clear, _Mapping]] = ..., - ) -> None: ... - -class ListStateCall(_message.Message): - __slots__ = ( - "stateName", - "exists", - "listStateGet", - "listStatePut", - "appendValue", - "appendList", - "clear", - ) - STATENAME_FIELD_NUMBER: _ClassVar[int] - EXISTS_FIELD_NUMBER: _ClassVar[int] - LISTSTATEGET_FIELD_NUMBER: _ClassVar[int] - LISTSTATEPUT_FIELD_NUMBER: _ClassVar[int] - APPENDVALUE_FIELD_NUMBER: _ClassVar[int] - APPENDLIST_FIELD_NUMBER: _ClassVar[int] - CLEAR_FIELD_NUMBER: _ClassVar[int] - stateName: str - exists: Exists - listStateGet: ListStateGet - listStatePut: ListStatePut - appendValue: AppendValue - appendList: AppendList - clear: Clear - def __init__( - self, - stateName: _Optional[str] = ..., - exists: _Optional[_Union[Exists, _Mapping]] = ..., - listStateGet: _Optional[_Union[ListStateGet, _Mapping]] = ..., - listStatePut: _Optional[_Union[ListStatePut, _Mapping]] = ..., - appendValue: _Optional[_Union[AppendValue, _Mapping]] = ..., - appendList: _Optional[_Union[AppendList, _Mapping]] = ..., - clear: _Optional[_Union[Clear, _Mapping]] = ..., - ) -> None: ... - -class MapStateCall(_message.Message): - __slots__ = ( - "stateName", - "exists", - "getValue", - "containsKey", - "updateValue", - "iterator", - "keys", - "values", - "removeKey", - "clear", - ) - STATENAME_FIELD_NUMBER: _ClassVar[int] - EXISTS_FIELD_NUMBER: _ClassVar[int] - GETVALUE_FIELD_NUMBER: _ClassVar[int] - CONTAINSKEY_FIELD_NUMBER: _ClassVar[int] - UPDATEVALUE_FIELD_NUMBER: _ClassVar[int] - ITERATOR_FIELD_NUMBER: _ClassVar[int] - KEYS_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] - REMOVEKEY_FIELD_NUMBER: _ClassVar[int] - CLEAR_FIELD_NUMBER: _ClassVar[int] - stateName: str - exists: Exists - getValue: GetValue - containsKey: ContainsKey - updateValue: UpdateValue - iterator: Iterator - keys: Keys - values: Values - removeKey: RemoveKey - clear: Clear - def __init__( - self, - stateName: _Optional[str] = ..., - exists: _Optional[_Union[Exists, _Mapping]] = ..., - getValue: _Optional[_Union[GetValue, _Mapping]] = ..., - containsKey: _Optional[_Union[ContainsKey, _Mapping]] = ..., - updateValue: _Optional[_Union[UpdateValue, _Mapping]] = ..., - iterator: _Optional[_Union[Iterator, _Mapping]] = ..., - keys: _Optional[_Union[Keys, _Mapping]] = ..., - values: _Optional[_Union[Values, _Mapping]] = ..., - removeKey: _Optional[_Union[RemoveKey, _Mapping]] = ..., - clear: _Optional[_Union[Clear, _Mapping]] = ..., - ) -> None: ... - -class SetImplicitKey(_message.Message): - __slots__ = ("key",) - KEY_FIELD_NUMBER: _ClassVar[int] - key: bytes - def __init__(self, key: _Optional[bytes] = ...) -> None: ... - -class RemoveImplicitKey(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class Exists(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class Get(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class RegisterTimer(_message.Message): - __slots__ = ("expiryTimestampMs",) - EXPIRYTIMESTAMPMS_FIELD_NUMBER: _ClassVar[int] - expiryTimestampMs: int - def __init__(self, expiryTimestampMs: _Optional[int] = ...) -> None: ... - -class DeleteTimer(_message.Message): - __slots__ = ("expiryTimestampMs",) - EXPIRYTIMESTAMPMS_FIELD_NUMBER: _ClassVar[int] - expiryTimestampMs: int - def __init__(self, expiryTimestampMs: _Optional[int] = ...) -> None: ... - -class ListTimers(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class ValueStateUpdate(_message.Message): - __slots__ = ("value",) - VALUE_FIELD_NUMBER: _ClassVar[int] - value: bytes - def __init__(self, value: _Optional[bytes] = ...) -> None: ... - -class Clear(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class ListStateGet(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class ListStatePut(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class AppendValue(_message.Message): - __slots__ = ("value",) - VALUE_FIELD_NUMBER: _ClassVar[int] - value: bytes - def __init__(self, value: _Optional[bytes] = ...) -> None: ... - -class AppendList(_message.Message): - __slots__ = () - def __init__(self) -> None: ... - -class GetValue(_message.Message): - __slots__ = ("userKey",) - USERKEY_FIELD_NUMBER: _ClassVar[int] - userKey: bytes - def __init__(self, userKey: _Optional[bytes] = ...) -> None: ... - -class ContainsKey(_message.Message): - __slots__ = ("userKey",) - USERKEY_FIELD_NUMBER: _ClassVar[int] - userKey: bytes - def __init__(self, userKey: _Optional[bytes] = ...) -> None: ... - -class UpdateValue(_message.Message): - __slots__ = ("userKey", "value") - USERKEY_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - userKey: bytes - value: bytes - def __init__(self, userKey: _Optional[bytes] = ..., value: _Optional[bytes] = ...) -> None: ... - -class Iterator(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class Keys(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class Values(_message.Message): - __slots__ = ("iteratorId",) - ITERATORID_FIELD_NUMBER: _ClassVar[int] - iteratorId: str - def __init__(self, iteratorId: _Optional[str] = ...) -> None: ... - -class RemoveKey(_message.Message): - __slots__ = ("userKey",) - USERKEY_FIELD_NUMBER: _ClassVar[int] - userKey: bytes - def __init__(self, userKey: _Optional[bytes] = ...) -> None: ... - -class SetHandleState(_message.Message): - __slots__ = ("state",) - STATE_FIELD_NUMBER: _ClassVar[int] - state: HandleState - def __init__(self, state: _Optional[_Union[HandleState, str]] = ...) -> None: ... - -class TTLConfig(_message.Message): - __slots__ = ("durationMs",) - DURATIONMS_FIELD_NUMBER: _ClassVar[int] - durationMs: int - def __init__(self, durationMs: _Optional[int] = ...) -> None: ... +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _HandleState: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _HandleStateEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_HandleState.ValueType], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + CREATED: _HandleState.ValueType # 0 + INITIALIZED: _HandleState.ValueType # 1 + DATA_PROCESSED: _HandleState.ValueType # 2 + TIMER_PROCESSED: _HandleState.ValueType # 3 + CLOSED: _HandleState.ValueType # 4 + +class HandleState(_HandleState, metaclass=_HandleStateEnumTypeWrapper): ... + +CREATED: HandleState.ValueType # 0 +INITIALIZED: HandleState.ValueType # 1 +DATA_PROCESSED: HandleState.ValueType # 2 +TIMER_PROCESSED: HandleState.ValueType # 3 +CLOSED: HandleState.ValueType # 4 +global___HandleState = HandleState + +class StateRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VERSION_FIELD_NUMBER: builtins.int + STATEFULPROCESSORCALL_FIELD_NUMBER: builtins.int + STATEVARIABLEREQUEST_FIELD_NUMBER: builtins.int + IMPLICITGROUPINGKEYREQUEST_FIELD_NUMBER: builtins.int + TIMERREQUEST_FIELD_NUMBER: builtins.int + version: builtins.int + @property + def statefulProcessorCall(self) -> global___StatefulProcessorCall: ... + @property + def stateVariableRequest(self) -> global___StateVariableRequest: ... + @property + def implicitGroupingKeyRequest(self) -> global___ImplicitGroupingKeyRequest: ... + @property + def timerRequest(self) -> global___TimerRequest: ... + def __init__( + self, + *, + version: builtins.int = ..., + statefulProcessorCall: global___StatefulProcessorCall | None = ..., + stateVariableRequest: global___StateVariableRequest | None = ..., + implicitGroupingKeyRequest: global___ImplicitGroupingKeyRequest | None = ..., + timerRequest: global___TimerRequest | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "implicitGroupingKeyRequest", + b"implicitGroupingKeyRequest", + "method", + b"method", + "stateVariableRequest", + b"stateVariableRequest", + "statefulProcessorCall", + b"statefulProcessorCall", + "timerRequest", + b"timerRequest", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "implicitGroupingKeyRequest", + b"implicitGroupingKeyRequest", + "method", + b"method", + "stateVariableRequest", + b"stateVariableRequest", + "statefulProcessorCall", + b"statefulProcessorCall", + "timerRequest", + b"timerRequest", + "version", + b"version", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> ( + typing_extensions.Literal[ + "statefulProcessorCall", + "stateVariableRequest", + "implicitGroupingKeyRequest", + "timerRequest", + ] + | None + ): ... + +global___StateRequest = StateRequest + +class StateResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATUSCODE_FIELD_NUMBER: builtins.int + ERRORMESSAGE_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + statusCode: builtins.int + errorMessage: builtins.str + value: builtins.bytes + def __init__( + self, + *, + statusCode: builtins.int = ..., + errorMessage: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "errorMessage", b"errorMessage", "statusCode", b"statusCode", "value", b"value" + ], + ) -> None: ... + +global___StateResponse = StateResponse + +class StateResponseWithLongTypeVal(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATUSCODE_FIELD_NUMBER: builtins.int + ERRORMESSAGE_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + statusCode: builtins.int + errorMessage: builtins.str + value: builtins.int + def __init__( + self, + *, + statusCode: builtins.int = ..., + errorMessage: builtins.str = ..., + value: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "errorMessage", b"errorMessage", "statusCode", b"statusCode", "value", b"value" + ], + ) -> None: ... + +global___StateResponseWithLongTypeVal = StateResponseWithLongTypeVal + +class StatefulProcessorCall(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SETHANDLESTATE_FIELD_NUMBER: builtins.int + GETVALUESTATE_FIELD_NUMBER: builtins.int + GETLISTSTATE_FIELD_NUMBER: builtins.int + GETMAPSTATE_FIELD_NUMBER: builtins.int + TIMERSTATECALL_FIELD_NUMBER: builtins.int + DELETEIFEXISTS_FIELD_NUMBER: builtins.int + @property + def setHandleState(self) -> global___SetHandleState: ... + @property + def getValueState(self) -> global___StateCallCommand: ... + @property + def getListState(self) -> global___StateCallCommand: ... + @property + def getMapState(self) -> global___StateCallCommand: ... + @property + def timerStateCall(self) -> global___TimerStateCallCommand: ... + @property + def deleteIfExists(self) -> global___StateCallCommand: ... + def __init__( + self, + *, + setHandleState: global___SetHandleState | None = ..., + getValueState: global___StateCallCommand | None = ..., + getListState: global___StateCallCommand | None = ..., + getMapState: global___StateCallCommand | None = ..., + timerStateCall: global___TimerStateCallCommand | None = ..., + deleteIfExists: global___StateCallCommand | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "deleteIfExists", + b"deleteIfExists", + "getListState", + b"getListState", + "getMapState", + b"getMapState", + "getValueState", + b"getValueState", + "method", + b"method", + "setHandleState", + b"setHandleState", + "timerStateCall", + b"timerStateCall", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "deleteIfExists", + b"deleteIfExists", + "getListState", + b"getListState", + "getMapState", + b"getMapState", + "getValueState", + b"getValueState", + "method", + b"method", + "setHandleState", + b"setHandleState", + "timerStateCall", + b"timerStateCall", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> ( + typing_extensions.Literal[ + "setHandleState", + "getValueState", + "getListState", + "getMapState", + "timerStateCall", + "deleteIfExists", + ] + | None + ): ... + +global___StatefulProcessorCall = StatefulProcessorCall + +class StateVariableRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VALUESTATECALL_FIELD_NUMBER: builtins.int + LISTSTATECALL_FIELD_NUMBER: builtins.int + MAPSTATECALL_FIELD_NUMBER: builtins.int + @property + def valueStateCall(self) -> global___ValueStateCall: ... + @property + def listStateCall(self) -> global___ListStateCall: ... + @property + def mapStateCall(self) -> global___MapStateCall: ... + def __init__( + self, + *, + valueStateCall: global___ValueStateCall | None = ..., + listStateCall: global___ListStateCall | None = ..., + mapStateCall: global___MapStateCall | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "listStateCall", + b"listStateCall", + "mapStateCall", + b"mapStateCall", + "method", + b"method", + "valueStateCall", + b"valueStateCall", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "listStateCall", + b"listStateCall", + "mapStateCall", + b"mapStateCall", + "method", + b"method", + "valueStateCall", + b"valueStateCall", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["valueStateCall", "listStateCall", "mapStateCall"] | None: ... + +global___StateVariableRequest = StateVariableRequest + +class ImplicitGroupingKeyRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SETIMPLICITKEY_FIELD_NUMBER: builtins.int + REMOVEIMPLICITKEY_FIELD_NUMBER: builtins.int + @property + def setImplicitKey(self) -> global___SetImplicitKey: ... + @property + def removeImplicitKey(self) -> global___RemoveImplicitKey: ... + def __init__( + self, + *, + setImplicitKey: global___SetImplicitKey | None = ..., + removeImplicitKey: global___RemoveImplicitKey | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "method", + b"method", + "removeImplicitKey", + b"removeImplicitKey", + "setImplicitKey", + b"setImplicitKey", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "method", + b"method", + "removeImplicitKey", + b"removeImplicitKey", + "setImplicitKey", + b"setImplicitKey", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["setImplicitKey", "removeImplicitKey"] | None: ... + +global___ImplicitGroupingKeyRequest = ImplicitGroupingKeyRequest + +class TimerRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TIMERVALUEREQUEST_FIELD_NUMBER: builtins.int + EXPIRYTIMERREQUEST_FIELD_NUMBER: builtins.int + @property + def timerValueRequest(self) -> global___TimerValueRequest: ... + @property + def expiryTimerRequest(self) -> global___ExpiryTimerRequest: ... + def __init__( + self, + *, + timerValueRequest: global___TimerValueRequest | None = ..., + expiryTimerRequest: global___ExpiryTimerRequest | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "expiryTimerRequest", + b"expiryTimerRequest", + "method", + b"method", + "timerValueRequest", + b"timerValueRequest", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "expiryTimerRequest", + b"expiryTimerRequest", + "method", + b"method", + "timerValueRequest", + b"timerValueRequest", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["timerValueRequest", "expiryTimerRequest"] | None: ... + +global___TimerRequest = TimerRequest + +class TimerValueRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + GETPROCESSINGTIMER_FIELD_NUMBER: builtins.int + GETWATERMARK_FIELD_NUMBER: builtins.int + @property + def getProcessingTimer(self) -> global___GetProcessingTime: ... + @property + def getWatermark(self) -> global___GetWatermark: ... + def __init__( + self, + *, + getProcessingTimer: global___GetProcessingTime | None = ..., + getWatermark: global___GetWatermark | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "getProcessingTimer", + b"getProcessingTimer", + "getWatermark", + b"getWatermark", + "method", + b"method", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "getProcessingTimer", + b"getProcessingTimer", + "getWatermark", + b"getWatermark", + "method", + b"method", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["getProcessingTimer", "getWatermark"] | None: ... + +global___TimerValueRequest = TimerValueRequest + +class ExpiryTimerRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EXPIRYTIMESTAMPMS_FIELD_NUMBER: builtins.int + expiryTimestampMs: builtins.int + def __init__( + self, + *, + expiryTimestampMs: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["expiryTimestampMs", b"expiryTimestampMs"] + ) -> None: ... + +global___ExpiryTimerRequest = ExpiryTimerRequest + +class GetProcessingTime(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___GetProcessingTime = GetProcessingTime + +class GetWatermark(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___GetWatermark = GetWatermark + +class StateCallCommand(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATENAME_FIELD_NUMBER: builtins.int + SCHEMA_FIELD_NUMBER: builtins.int + MAPSTATEVALUESCHEMA_FIELD_NUMBER: builtins.int + TTL_FIELD_NUMBER: builtins.int + stateName: builtins.str + schema: builtins.str + mapStateValueSchema: builtins.str + @property + def ttl(self) -> global___TTLConfig: ... + def __init__( + self, + *, + stateName: builtins.str = ..., + schema: builtins.str = ..., + mapStateValueSchema: builtins.str = ..., + ttl: global___TTLConfig | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["ttl", b"ttl"]) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "mapStateValueSchema", + b"mapStateValueSchema", + "schema", + b"schema", + "stateName", + b"stateName", + "ttl", + b"ttl", + ], + ) -> None: ... + +global___StateCallCommand = StateCallCommand + +class TimerStateCallCommand(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REGISTER_FIELD_NUMBER: builtins.int + DELETE_FIELD_NUMBER: builtins.int + LIST_FIELD_NUMBER: builtins.int + @property + def register(self) -> global___RegisterTimer: ... + @property + def delete(self) -> global___DeleteTimer: ... + @property + def list(self) -> global___ListTimers: ... + def __init__( + self, + *, + register: global___RegisterTimer | None = ..., + delete: global___DeleteTimer | None = ..., + list: global___ListTimers | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "delete", b"delete", "list", b"list", "method", b"method", "register", b"register" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "delete", b"delete", "list", b"list", "method", b"method", "register", b"register" + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["register", "delete", "list"] | None: ... + +global___TimerStateCallCommand = TimerStateCallCommand + +class ValueStateCall(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATENAME_FIELD_NUMBER: builtins.int + EXISTS_FIELD_NUMBER: builtins.int + GET_FIELD_NUMBER: builtins.int + VALUESTATEUPDATE_FIELD_NUMBER: builtins.int + CLEAR_FIELD_NUMBER: builtins.int + stateName: builtins.str + @property + def exists(self) -> global___Exists: ... + @property + def get(self) -> global___Get: ... + @property + def valueStateUpdate(self) -> global___ValueStateUpdate: ... + @property + def clear(self) -> global___Clear: ... + def __init__( + self, + *, + stateName: builtins.str = ..., + exists: global___Exists | None = ..., + get: global___Get | None = ..., + valueStateUpdate: global___ValueStateUpdate | None = ..., + clear: global___Clear | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "clear", + b"clear", + "exists", + b"exists", + "get", + b"get", + "method", + b"method", + "valueStateUpdate", + b"valueStateUpdate", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "clear", + b"clear", + "exists", + b"exists", + "get", + b"get", + "method", + b"method", + "stateName", + b"stateName", + "valueStateUpdate", + b"valueStateUpdate", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> typing_extensions.Literal["exists", "get", "valueStateUpdate", "clear"] | None: ... + +global___ValueStateCall = ValueStateCall + +class ListStateCall(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATENAME_FIELD_NUMBER: builtins.int + EXISTS_FIELD_NUMBER: builtins.int + LISTSTATEGET_FIELD_NUMBER: builtins.int + LISTSTATEPUT_FIELD_NUMBER: builtins.int + APPENDVALUE_FIELD_NUMBER: builtins.int + APPENDLIST_FIELD_NUMBER: builtins.int + CLEAR_FIELD_NUMBER: builtins.int + stateName: builtins.str + @property + def exists(self) -> global___Exists: ... + @property + def listStateGet(self) -> global___ListStateGet: ... + @property + def listStatePut(self) -> global___ListStatePut: ... + @property + def appendValue(self) -> global___AppendValue: ... + @property + def appendList(self) -> global___AppendList: ... + @property + def clear(self) -> global___Clear: ... + def __init__( + self, + *, + stateName: builtins.str = ..., + exists: global___Exists | None = ..., + listStateGet: global___ListStateGet | None = ..., + listStatePut: global___ListStatePut | None = ..., + appendValue: global___AppendValue | None = ..., + appendList: global___AppendList | None = ..., + clear: global___Clear | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "appendList", + b"appendList", + "appendValue", + b"appendValue", + "clear", + b"clear", + "exists", + b"exists", + "listStateGet", + b"listStateGet", + "listStatePut", + b"listStatePut", + "method", + b"method", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "appendList", + b"appendList", + "appendValue", + b"appendValue", + "clear", + b"clear", + "exists", + b"exists", + "listStateGet", + b"listStateGet", + "listStatePut", + b"listStatePut", + "method", + b"method", + "stateName", + b"stateName", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> ( + typing_extensions.Literal[ + "exists", "listStateGet", "listStatePut", "appendValue", "appendList", "clear" + ] + | None + ): ... + +global___ListStateCall = ListStateCall + +class MapStateCall(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATENAME_FIELD_NUMBER: builtins.int + EXISTS_FIELD_NUMBER: builtins.int + GETVALUE_FIELD_NUMBER: builtins.int + CONTAINSKEY_FIELD_NUMBER: builtins.int + UPDATEVALUE_FIELD_NUMBER: builtins.int + ITERATOR_FIELD_NUMBER: builtins.int + KEYS_FIELD_NUMBER: builtins.int + VALUES_FIELD_NUMBER: builtins.int + REMOVEKEY_FIELD_NUMBER: builtins.int + CLEAR_FIELD_NUMBER: builtins.int + stateName: builtins.str + @property + def exists(self) -> global___Exists: ... + @property + def getValue(self) -> global___GetValue: ... + @property + def containsKey(self) -> global___ContainsKey: ... + @property + def updateValue(self) -> global___UpdateValue: ... + @property + def iterator(self) -> global___Iterator: ... + @property + def keys(self) -> global___Keys: ... + @property + def values(self) -> global___Values: ... + @property + def removeKey(self) -> global___RemoveKey: ... + @property + def clear(self) -> global___Clear: ... + def __init__( + self, + *, + stateName: builtins.str = ..., + exists: global___Exists | None = ..., + getValue: global___GetValue | None = ..., + containsKey: global___ContainsKey | None = ..., + updateValue: global___UpdateValue | None = ..., + iterator: global___Iterator | None = ..., + keys: global___Keys | None = ..., + values: global___Values | None = ..., + removeKey: global___RemoveKey | None = ..., + clear: global___Clear | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "clear", + b"clear", + "containsKey", + b"containsKey", + "exists", + b"exists", + "getValue", + b"getValue", + "iterator", + b"iterator", + "keys", + b"keys", + "method", + b"method", + "removeKey", + b"removeKey", + "updateValue", + b"updateValue", + "values", + b"values", + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "clear", + b"clear", + "containsKey", + b"containsKey", + "exists", + b"exists", + "getValue", + b"getValue", + "iterator", + b"iterator", + "keys", + b"keys", + "method", + b"method", + "removeKey", + b"removeKey", + "stateName", + b"stateName", + "updateValue", + b"updateValue", + "values", + b"values", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["method", b"method"] + ) -> ( + typing_extensions.Literal[ + "exists", + "getValue", + "containsKey", + "updateValue", + "iterator", + "keys", + "values", + "removeKey", + "clear", + ] + | None + ): ... + +global___MapStateCall = MapStateCall + +class SetImplicitKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + key: builtins.bytes + def __init__( + self, + *, + key: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key"]) -> None: ... + +global___SetImplicitKey = SetImplicitKey + +class RemoveImplicitKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___RemoveImplicitKey = RemoveImplicitKey + +class Exists(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___Exists = Exists + +class Get(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___Get = Get + +class RegisterTimer(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EXPIRYTIMESTAMPMS_FIELD_NUMBER: builtins.int + expiryTimestampMs: builtins.int + def __init__( + self, + *, + expiryTimestampMs: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["expiryTimestampMs", b"expiryTimestampMs"] + ) -> None: ... + +global___RegisterTimer = RegisterTimer + +class DeleteTimer(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EXPIRYTIMESTAMPMS_FIELD_NUMBER: builtins.int + expiryTimestampMs: builtins.int + def __init__( + self, + *, + expiryTimestampMs: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["expiryTimestampMs", b"expiryTimestampMs"] + ) -> None: ... + +global___DeleteTimer = DeleteTimer + +class ListTimers(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___ListTimers = ListTimers + +class ValueStateUpdate(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VALUE_FIELD_NUMBER: builtins.int + value: builtins.bytes + def __init__( + self, + *, + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ... + +global___ValueStateUpdate = ValueStateUpdate + +class Clear(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___Clear = Clear + +class ListStateGet(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___ListStateGet = ListStateGet + +class ListStatePut(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___ListStatePut = ListStatePut + +class AppendValue(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VALUE_FIELD_NUMBER: builtins.int + value: builtins.bytes + def __init__( + self, + *, + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ... + +global___AppendValue = AppendValue + +class AppendList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___AppendList = AppendList + +class GetValue(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + USERKEY_FIELD_NUMBER: builtins.int + userKey: builtins.bytes + def __init__( + self, + *, + userKey: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["userKey", b"userKey"]) -> None: ... + +global___GetValue = GetValue + +class ContainsKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + USERKEY_FIELD_NUMBER: builtins.int + userKey: builtins.bytes + def __init__( + self, + *, + userKey: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["userKey", b"userKey"]) -> None: ... + +global___ContainsKey = ContainsKey + +class UpdateValue(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + USERKEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + userKey: builtins.bytes + value: builtins.bytes + def __init__( + self, + *, + userKey: builtins.bytes = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["userKey", b"userKey", "value", b"value"] + ) -> None: ... + +global___UpdateValue = UpdateValue + +class Iterator(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___Iterator = Iterator + +class Keys(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___Keys = Keys + +class Values(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ITERATORID_FIELD_NUMBER: builtins.int + iteratorId: builtins.str + def __init__( + self, + *, + iteratorId: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["iteratorId", b"iteratorId"] + ) -> None: ... + +global___Values = Values + +class RemoveKey(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + USERKEY_FIELD_NUMBER: builtins.int + userKey: builtins.bytes + def __init__( + self, + *, + userKey: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["userKey", b"userKey"]) -> None: ... + +global___RemoveKey = RemoveKey + +class SetHandleState(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STATE_FIELD_NUMBER: builtins.int + state: global___HandleState.ValueType + def __init__( + self, + *, + state: global___HandleState.ValueType = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["state", b"state"]) -> None: ... + +global___SetHandleState = SetHandleState + +class TTLConfig(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DURATIONMS_FIELD_NUMBER: builtins.int + durationMs: builtins.int + def __init__( + self, + *, + durationMs: builtins.int = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["durationMs", b"durationMs"] + ) -> None: ... + +global___TTLConfig = TTLConfig From 871a3a9c55ba7bf81b135ef97d1df63841b55e3f Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 11 Nov 2024 11:32:05 +0800 Subject: [PATCH 08/14] use one script --- .github/workflows/build_and_test.yml | 10 +-- ...onnect-check-protos.py => check-protos.py} | 23 ++--- dev/connect-gen-protos.sh | 78 +---------------- ...ming-state-gen-protos.sh => gen-protos.sh} | 70 +++++++++++----- dev/streaming-gen-protos.sh | 27 ++++++ dev/streaming-state-check-protos.py | 84 ------------------- 6 files changed, 94 insertions(+), 198 deletions(-) rename dev/{connect-check-protos.py => check-protos.py} (73%) rename dev/{streaming-state-gen-protos.sh => gen-protos.sh} (62%) create mode 100755 dev/streaming-gen-protos.sh delete mode 100755 dev/streaming-state-check-protos.py diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9754f7acfdde..16aa33f681fb 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -648,12 +648,12 @@ jobs: run: | python3.11 -m pip install 'black==23.9.1' 'protobuf==5.28.3' 'mypy==1.8.0' 'mypy-protobuf==3.3.0' python3.11 -m pip list + - name: Python CodeGen check for branch-3.5 + if: inputs.branch == 'branch-3.5' + run: ./dev/connect-check-protos.py - name: Python CodeGen check - run: | - ./dev/connect-check-protos.py - if [ -f ./dev/streaming-state-check-protos.py ]; then - ./dev/streaming-state-check-protos.py - fi + if: inputs.branch != 'branch-3.5' + run: ./dev/check-protos.py # Static analysis lint: diff --git a/dev/connect-check-protos.py b/dev/check-protos.py similarity index 73% rename from dev/connect-check-protos.py rename to dev/check-protos.py index 9ba56bae6b19..bfca8b27be21 100755 --- a/dev/connect-check-protos.py +++ b/dev/check-protos.py @@ -18,7 +18,7 @@ # # Utility for checking whether generated codes in PySpark are out of sync. -# usage: ./dev/connect-check-protos.py +# usage: ./dev/check-protos.py import os import sys @@ -43,12 +43,12 @@ def run_cmd(cmd): return subprocess.check_output(cmd.split(" ")).decode("utf-8") -def check_connect_protos(): - print("Start checking the generated codes in pyspark-connect.") - with tempfile.TemporaryDirectory(prefix="check_connect_protos") as tmp: - run_cmd(f"{SPARK_HOME}/dev/connect-gen-protos.sh {tmp}") +def check_protos(module_name, cmp_path, proto_path): + print(f"Start checking the generated codes in pyspark-${module_name}.") + with tempfile.TemporaryDirectory(prefix=f"check_${module_name}__protos") as tmp: + run_cmd(f"{SPARK_HOME}/dev/gen-protos.sh {module_name} {tmp}") result = filecmp.dircmp( - f"{SPARK_HOME}/python/pyspark/sql/connect/proto/", + f"{SPARK_HOME}/{cmp_path}", tmp, ignore=["__init__.py", "__pycache__"], ) @@ -71,14 +71,17 @@ def check_connect_protos(): success = False if success: - print("Finish checking the generated codes in pyspark-connect: SUCCESS") + print(f"Finish checking the generated codes in pyspark-${module_name}: SUCCESS") else: fail( "Generated files for pyspark-connect are out of sync! " - "If you have touched files under sql/connect/common/src/main/protobuf/, " - "please run ./dev/connect-gen-protos.sh. " + f"If you have touched files under ${proto_path}, " + f"please run ./dev/${module_name}-gen-protos.sh. " "If you haven't touched any file above, please rebase your PR against main branch." ) -check_connect_protos() +check_protos( + "connect", "python/pyspark/sql/connect/proto/", "sql/connect/common/src/main/protobuf/" +) +check_protos("streaming", "python/pyspark/sql/streaming/proto/", "sql/core/src/main/protobuf/") diff --git a/dev/connect-gen-protos.sh b/dev/connect-gen-protos.sh index 2805908890ee..8ed323cc4259 100755 --- a/dev/connect-gen-protos.sh +++ b/dev/connect-gen-protos.sh @@ -24,80 +24,4 @@ if [[ $# -gt 1 ]]; then exit -1 fi - -SPARK_HOME="$(cd "`dirname $0`"/..; pwd)" -cd "$SPARK_HOME" - - -OUTPUT_PATH=${SPARK_HOME}/python/pyspark/sql/connect/proto/ -if [[ $# -eq 1 ]]; then - rm -Rf $1 - mkdir -p $1 - OUTPUT_PATH=$1 -fi - -pushd sql/connect/common/src/main - -LICENSE=$(cat <<'EOF' -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -EOF) -echo "$LICENSE" > /tmp/tmp_licence - - -# Delete the old generated protobuf files. -rm -Rf gen - -# Now, regenerate the new files -buf generate --debug -vvv - -# We need to edit the generate python files to account for the actual package location and not -# the one generated by proto. -for f in `find gen/proto/python -name "*.py*"`; do - # First fix the imports. - if [[ $f == *_pb2.py || $f == *_pb2_grpc.py ]]; then - sed -e 's/from spark.connect import/from pyspark.sql.connect.proto import/g' $f > $f.tmp - mv $f.tmp $f - # Now fix the module name in the serialized descriptor. - sed -e "s/DESCRIPTOR, 'spark.connect/DESCRIPTOR, 'pyspark.sql.connect.proto/g" $f > $f.tmp - mv $f.tmp $f - elif [[ $f == *.pyi ]]; then - sed -e 's/import spark.connect./import pyspark.sql.connect.proto./g' -e 's/spark.connect./pyspark.sql.connect.proto./g' -e '/ *@typing_extensions\.final/d' $f > $f.tmp - mv $f.tmp $f - fi - - # Prepend the Apache licence header to the files. - cp $f $f.bak - cat /tmp/tmp_licence $f.bak > $f - - LC=$(wc -l < $f) - echo $LC - if [[ $f == *_grpc.py && $LC -eq 20 ]]; then - rm $f - fi - rm $f.bak -done - -black --config $SPARK_HOME/dev/pyproject.toml gen/proto/python - -# Last step copy the result files to the destination module. -for f in `find gen/proto/python -name "*.py*"`; do - cp $f $OUTPUT_PATH -done - -# Clean up everything. -rm -Rf gen +./dev/gen-protos.sh connect "$@" diff --git a/dev/streaming-state-gen-protos.sh b/dev/gen-protos.sh similarity index 62% rename from dev/streaming-state-gen-protos.sh rename to dev/gen-protos.sh index b421e9751f32..faad6f4de512 100755 --- a/dev/streaming-state-gen-protos.sh +++ b/dev/gen-protos.sh @@ -16,27 +16,45 @@ # See the License for the specific language governing permissions and # limitations under the License. # -set -ex - -if [[ $# -gt 1 ]]; then - echo "Illegal number of parameters." - echo "Usage: ./dev/stream-state-gen-protos.sh [path]" - exit -1 -fi +set -ex SPARK_HOME="$(cd "`dirname $0`"/..; pwd)" cd "$SPARK_HOME" +OUTPUT_PATH="" +MODULE="" -OUTPUT_PATH=${SPARK_HOME}/python/pyspark/sql/streaming/proto -if [[ $# -eq 1 ]]; then - rm -Rf $1 - mkdir -p $1 - OUTPUT_PATH=$1 +function usage() { + echo "Usage:./dev/gen-protos.sh [connect|streaming] [output_path]" + exit -1 +} + +if [[ $# -lt 1 || $# -gt 2 ]]; then + usage fi -pushd sql/core/src/main +if [[ $1 == "connect" ]]; then + MODULE="connect" + OUTPUT_PATH=${SPARK_HOME}/python/pyspark/sql/connect/proto/ +elif [[ $1 == "streaming" ]]; then + MODULE="streaming" + OUTPUT_PATH=${SPARK_HOME}/python/pyspark/sql/streaming/proto/ +else + usage +fi + +if [[ $# -eq 2 ]]; then + rm -Rf $2 + mkdir -p $2 + OUTPUT_PATH=$2 +fi + +if [[ $MODULE == "connect" ]]; then + pushd sql/connect/common/src/main +elif [[ $MODULE == "streaming" ]]; then + pushd sql/core/src/main +fi LICENSE=$(cat <<'EOF' # @@ -58,25 +76,29 @@ LICENSE=$(cat <<'EOF' EOF) echo "$LICENSE" > /tmp/tmp_licence - # Delete the old generated protobuf files. rm -Rf gen # Now, regenerate the new files buf generate --debug -vvv -# We need to edit the generate python files to account for the actual package location and not -# the one generated by proto. for f in `find gen/proto/python -name "*.py*"`; do - # First fix the imports. - if [[ $f == *_pb2.py ]]; then + if [[ $MODULE == "connect" && ($f == *_pb2.py || $f == *_pb2_grpc.py) ]]; then + sed -e 's/from spark.connect import/from pyspark.sql.connect.proto import/g' $f > $f.tmp + mv $f.tmp $f + sed -e "s/DESCRIPTOR, 'spark.connect/DESCRIPTOR, 'pyspark.sql.connect.proto/g" $f > $f.tmp + mv $f.tmp $f + elif [[ $MODULE == "streaming" && $f == *_pb2.py ]]; then sed -e 's/from spark.streaming import/from pyspark.sql.streaming.proto import/g' $f > $f.tmp mv $f.tmp $f - # Now fix the module name in the serialized descriptor. sed -e "s/DESCRIPTOR, 'spark.streaming/DESCRIPTOR, 'pyspark.sql.streaming.proto/g" $f > $f.tmp mv $f.tmp $f elif [[ $f == *.pyi ]]; then - sed -e 's/import spark.streaming./import pyspark.sql.streaming.proto./g' -e 's/spark.streaming./pyspark.sql.streaming.proto./g' -e '/ *@typing_extensions\.final/d' $f > $f.tmp + if [[ $MODULE == "connect" ]]; then + sed -e 's/import spark.connect./import pyspark.sql.connect.proto./g' -e 's/spark.connect./pyspark.sql.connect.proto./g' -e '/ *@typing_extensions\.final/d' $f > $f.tmp + else + sed -e 's/import spark.streaming./import pyspark.sql.streaming.proto./g' -e 's/spark.streaming./pyspark.sql.streaming.proto./g' -e '/ *@typing_extensions\.final/d' $f > $f.tmp + fi mv $f.tmp $f fi @@ -84,15 +106,19 @@ for f in `find gen/proto/python -name "*.py*"`; do cp $f $f.bak cat /tmp/tmp_licence $f.bak > $f + LC=$(wc -l < $f) + echo $LC + if [[ $f == *_grpc.py && $LC -eq 20 ]]; then + rm $f + fi rm $f.bak done black --config $SPARK_HOME/dev/pyproject.toml gen/proto/python -# Last step copy the result files to the destination module. for f in `find gen/proto/python -name "*.py*"`; do cp $f $OUTPUT_PATH done # Clean up everything. -rm -Rf gen +rm -Rf gen \ No newline at end of file diff --git a/dev/streaming-gen-protos.sh b/dev/streaming-gen-protos.sh new file mode 100755 index 000000000000..3d80bda4fb94 --- /dev/null +++ b/dev/streaming-gen-protos.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -ex + +if [[ $# -gt 1 ]]; then + echo "Illegal number of parameters." + echo "Usage: ./dev/streaming-gen-protos.sh [path]" + exit -1 +fi + +./dev/gen-protos.sh streaming "$@" diff --git a/dev/streaming-state-check-protos.py b/dev/streaming-state-check-protos.py deleted file mode 100755 index 2b83c4ab0af6..000000000000 --- a/dev/streaming-state-check-protos.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Utility for checking whether generated codes in PySpark are out of sync. -# usage: ./dev/streaming-state-gen-protos-check.py - -import os -import sys -import filecmp -import tempfile -import subprocess - -# Location of your Spark git development area -SPARK_HOME = os.environ.get("SPARK_HOME", os.getcwd()) - - -def fail(msg): - print(msg) - sys.exit(-1) - - -def run_cmd(cmd): - print(f"RUN: {cmd}") - if isinstance(cmd, list): - return subprocess.check_output(cmd).decode("utf-8") - else: - return subprocess.check_output(cmd.split(" ")).decode("utf-8") - - -def check_streaming_protos(): - print("Start checking the generated codes in pyspark-streaming.") - with tempfile.TemporaryDirectory(prefix="check_streaming_protos") as tmp: - run_cmd(f"{SPARK_HOME}/dev/streaming-state-gen-protos.sh {tmp}") - result = filecmp.dircmp( - f"{SPARK_HOME}/python/pyspark/sql/streaming/proto/", - tmp, - ignore=["__init__.py", "__pycache__"], - ) - success = True - - if len(result.left_only) > 0: - print(f"Unexpected files: {result.left_only}") - success = False - - if len(result.right_only) > 0: - print(f"Missing files: {result.right_only}") - success = False - - if len(result.funny_files) > 0: - print(f"Incomparable files: {result.funny_files}") - success = False - - if len(result.diff_files) > 0: - print(f"Different files: {result.diff_files}") - success = False - - if success: - print("Finish checking the generated codes in pyspark-streaming: SUCCESS") - else: - fail( - "Generated files for pyspark-streaming are out of sync! " - "If you have touched files under sql/core/src/main/protobuf/, " - "please run ./dev/streaming-state-gen-protos.sh. " - "If you haven't touched any file above, please rebase your PR against main branch." - ) - - -check_streaming_protos() From 05606ef25fabd22f63d99dcdbdfb36ecca242175 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 11 Nov 2024 13:12:15 +0800 Subject: [PATCH 09/14] refactor gen-protos.sh --- dev/gen-protos.sh | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/dev/gen-protos.sh b/dev/gen-protos.sh index faad6f4de512..496a5d44547a 100755 --- a/dev/gen-protos.sh +++ b/dev/gen-protos.sh @@ -24,9 +24,11 @@ cd "$SPARK_HOME" OUTPUT_PATH="" MODULE="" +SOURCE_MODULE="" +TARGET_MODULE="" function usage() { - echo "Usage:./dev/gen-protos.sh [connect|streaming] [output_path]" + echo "Usage:./dev/gen-protos.sh [connect|streaming] [output_path]" exit -1 } @@ -37,9 +39,13 @@ fi if [[ $1 == "connect" ]]; then MODULE="connect" OUTPUT_PATH=${SPARK_HOME}/python/pyspark/sql/connect/proto/ + SOURCE_MODULE="spark.connect" + TARGET_MODULE="pyspark.sql.connect.proto" elif [[ $1 == "streaming" ]]; then MODULE="streaming" OUTPUT_PATH=${SPARK_HOME}/python/pyspark/sql/streaming/proto/ + SOURCE_MODULE="spark.streaming" + TARGET_MODULE="pyspark.sql.streaming.proto" else usage fi @@ -82,23 +88,18 @@ rm -Rf gen # Now, regenerate the new files buf generate --debug -vvv +# We need to edit the generate python files to account for the actual package location and not +# the one generated by proto. for f in `find gen/proto/python -name "*.py*"`; do - if [[ $MODULE == "connect" && ($f == *_pb2.py || $f == *_pb2_grpc.py) ]]; then - sed -e 's/from spark.connect import/from pyspark.sql.connect.proto import/g' $f > $f.tmp + # First fix the imports. + if [[ $f == *_pb2.py || $f == *_pb2_grpc.py ]]; then + sed -e "s/from ${SOURCE_MODULE} import/from ${TARGET_MODULE} import/g" $f > $f.tmp mv $f.tmp $f - sed -e "s/DESCRIPTOR, 'spark.connect/DESCRIPTOR, 'pyspark.sql.connect.proto/g" $f > $f.tmp - mv $f.tmp $f - elif [[ $MODULE == "streaming" && $f == *_pb2.py ]]; then - sed -e 's/from spark.streaming import/from pyspark.sql.streaming.proto import/g' $f > $f.tmp - mv $f.tmp $f - sed -e "s/DESCRIPTOR, 'spark.streaming/DESCRIPTOR, 'pyspark.sql.streaming.proto/g" $f > $f.tmp + # Now fix the module name in the serialized descriptor. + sed -e "s/DESCRIPTOR, '${SOURCE_MODULE}/DESCRIPTOR, '${TARGET_MODULE}/g" $f > $f.tmp mv $f.tmp $f elif [[ $f == *.pyi ]]; then - if [[ $MODULE == "connect" ]]; then - sed -e 's/import spark.connect./import pyspark.sql.connect.proto./g' -e 's/spark.connect./pyspark.sql.connect.proto./g' -e '/ *@typing_extensions\.final/d' $f > $f.tmp - else - sed -e 's/import spark.streaming./import pyspark.sql.streaming.proto./g' -e 's/spark.streaming./pyspark.sql.streaming.proto./g' -e '/ *@typing_extensions\.final/d' $f > $f.tmp - fi + sed -e "s/import ${SOURCE_MODULE}./import ${TARGET_MODULE}./g" -e "s/${SOURCE_MODULE}./${TARGET_MODULE}./g" -e '/ *@typing_extensions\.final/d' $f > $f.tmp mv $f.tmp $f fi From 24dc9d1c5172119d9e0343e5a2dda96739c64eee Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 11 Nov 2024 13:20:26 +0800 Subject: [PATCH 10/14] fix streaming --- dev/gen-protos.sh | 2 +- python/pyspark/sql/streaming/proto/StateMessage_pb2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/gen-protos.sh b/dev/gen-protos.sh index 496a5d44547a..f6d0a46bafe5 100755 --- a/dev/gen-protos.sh +++ b/dev/gen-protos.sh @@ -44,7 +44,7 @@ if [[ $1 == "connect" ]]; then elif [[ $1 == "streaming" ]]; then MODULE="streaming" OUTPUT_PATH=${SPARK_HOME}/python/pyspark/sql/streaming/proto/ - SOURCE_MODULE="spark.streaming" + SOURCE_MODULE="org.apache.spark.sql.execution.streaming" TARGET_MODULE="pyspark.sql.streaming.proto" else usage diff --git a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py index a521e2968007..0a54690513a3 100644 --- a/python/pyspark/sql/streaming/proto/StateMessage_pb2.py +++ b/python/pyspark/sql/streaming/proto/StateMessage_pb2.py @@ -46,7 +46,7 @@ _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages( - DESCRIPTOR, "org.apache.spark.sql.execution.streaming.StateMessage_pb2", _globals + DESCRIPTOR, "pyspark.sql.streaming.proto.StateMessage_pb2", _globals ) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None From 8c6e3db0525cc7fe46c09684e0c0442c7de9ebe7 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 11 Nov 2024 13:27:10 +0800 Subject: [PATCH 11/14] add blank line --- dev/gen-protos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/gen-protos.sh b/dev/gen-protos.sh index f6d0a46bafe5..1a23a7caaee4 100755 --- a/dev/gen-protos.sh +++ b/dev/gen-protos.sh @@ -122,4 +122,4 @@ for f in `find gen/proto/python -name "*.py*"`; do done # Clean up everything. -rm -Rf gen \ No newline at end of file +rm -Rf gen From 29f8aaa68a370096078b0ced14b579895381b90d Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 11 Nov 2024 21:41:50 +0800 Subject: [PATCH 12/14] indentation --- dev/gen-protos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/gen-protos.sh b/dev/gen-protos.sh index 1a23a7caaee4..5e3832b93a1a 100755 --- a/dev/gen-protos.sh +++ b/dev/gen-protos.sh @@ -28,7 +28,7 @@ SOURCE_MODULE="" TARGET_MODULE="" function usage() { - echo "Usage:./dev/gen-protos.sh [connect|streaming] [output_path]" + echo "Usage:./dev/gen-protos.sh [connect|streaming] [output_path]" exit -1 } From 0bac0e34c639b383b2e84ff2ff90476cad696fca Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 11 Nov 2024 21:46:20 +0800 Subject: [PATCH 13/14] add echo --- dev/gen-protos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/gen-protos.sh b/dev/gen-protos.sh index 5e3832b93a1a..f0782d8d6d0c 100755 --- a/dev/gen-protos.sh +++ b/dev/gen-protos.sh @@ -28,6 +28,7 @@ SOURCE_MODULE="" TARGET_MODULE="" function usage() { + echo "Illegal number of parameters." echo "Usage:./dev/gen-protos.sh [connect|streaming] [output_path]" exit -1 } From 3837dc71061690397ea76246dfedefd23348a782 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 11 Nov 2024 21:48:35 +0800 Subject: [PATCH 14/14] add echo --- dev/gen-protos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/gen-protos.sh b/dev/gen-protos.sh index f0782d8d6d0c..d169964feb85 100755 --- a/dev/gen-protos.sh +++ b/dev/gen-protos.sh @@ -118,6 +118,7 @@ done black --config $SPARK_HOME/dev/pyproject.toml gen/proto/python +# Last step copy the result files to the destination module. for f in `find gen/proto/python -name "*.py*"`; do cp $f $OUTPUT_PATH done