diff --git a/docs/examples/api-reference/sources/source.py b/docs/examples/api-reference/sources/source.py
index 79949e223..7b384ea70 100644
--- a/docs/examples/api-reference/sources/source.py
+++ b/docs/examples/api-reference/sources/source.py
@@ -2,6 +2,7 @@
import sys
from datetime import datetime
+from fennel.expr import col
from fennel.testing import mock
__owner__ = "nikhil@fennel.ai"
@@ -41,6 +42,7 @@ def test_source_decorator(client):
schema={"age": str},
), # converting age dtype to int
},
+ where=eval(col("age") >= 18, schema={"age": int}),
env="prod",
sample=Sample(0.2, using=["email"]),
bounded=True,
diff --git a/docs/examples/concepts/source.py b/docs/examples/concepts/source.py
index 1ef9be7c3..e9cc25599 100644
--- a/docs/examples/concepts/source.py
+++ b/docs/examples/concepts/source.py
@@ -1,8 +1,9 @@
import sys
from datetime import datetime
-from fennel.connectors import Kafka, S3, source
+from fennel.connectors import Kafka, S3, eval, source
from fennel.datasets import dataset
+from fennel.expr import col
from fennel.testing import mock
__owner__ = "owner@example.com"
@@ -70,7 +71,7 @@ class Order:
@mock
-def test_filter_preproc_source(client):
+def test_where_source(client):
if sys.version_info >= (3, 10):
s3 = S3(name="mys3")
@@ -94,3 +95,30 @@ class Order:
datasets=[Order],
)
# /docsnip
+
+
+@mock
+def test_expression_where_source(client):
+ if sys.version_info >= (3, 10):
+ s3 = S3(name="mys3")
+
+ # docsnip where
+ # docsnip-highlight start
+ @source(
+ s3.bucket("data", path="*"),
+ disorder="1w",
+ cdc="append",
+ where=eval(col("skuid") >= 1000, schema={"skuid": int}),
+ )
+ # docsnip-highlight end
+ @dataset
+ class Order:
+ uid: int
+ skuid: int
+ at: datetime
+
+ client.commit(
+ message="some commit msg",
+ datasets=[Order],
+ )
+ # /docsnip
diff --git a/docs/pages/api-reference/decorators/source.md b/docs/pages/api-reference/decorators/source.md
index f42da5616..675a250ea 100644
--- a/docs/pages/api-reference/decorators/source.md
+++ b/docs/pages/api-reference/decorators/source.md
@@ -127,6 +127,15 @@ for other parent types apart from struct, please reach out to Fennel support.
+
+When present, filters source dataset rows with the input value.
+
+As of now there are two kinds of values of where:
+* `Callable`: In this case the input is a lambda which is used to filter rows.
+* `Eval`: Similar to eval value in preproc, the input here is an expression which is
+ used to filter rows.
+
+
When not set or set as `False`, it indicates that the source possesses infinite
diff --git a/docs/pages/concepts/source.md b/docs/pages/concepts/source.md
index bfffb4e22..8076a05c1 100644
--- a/docs/pages/concepts/source.md
+++ b/docs/pages/concepts/source.md
@@ -178,10 +178,11 @@ of that column on another column.
### Where
The `where` field in the source provides a way to filter out some rows from source
-during ingestion. Provide the filter criteria as a lambda function.
+during ingestion. Provide the filter criteria as a lambda function or an
+expression.
### Since
diff --git a/fennel/CHANGELOG.md b/fennel/CHANGELOG.md
index d0f39ef64..39c1b5c4c 100644
--- a/fennel/CHANGELOG.md
+++ b/fennel/CHANGELOG.md
@@ -1,4 +1,8 @@
# Changelog
+
+## [1.5.52] - 2024-11-12
+- Source: support expressions in where
+
## [1.5.51] - 2024-11-09
- Fix bug in mock client for session based dedup operator
diff --git a/fennel/client_tests/test_preproc_filter.py b/fennel/client_tests/test_where.py
similarity index 60%
rename from fennel/client_tests/test_preproc_filter.py
rename to fennel/client_tests/test_where.py
index 714ec8829..3ff702ad3 100644
--- a/fennel/client_tests/test_preproc_filter.py
+++ b/fennel/client_tests/test_where.py
@@ -4,13 +4,14 @@
import pandas as pd
import pytest
-from fennel.connectors import Webhook, source
+from fennel.connectors import Webhook, eval, source
from fennel.datasets import (
Dataset,
dataset,
field,
pipeline,
)
+from fennel.expr import col
from fennel.lib import inputs, meta
from fennel.testing import mock
@@ -20,7 +21,7 @@
@pytest.mark.integration
@mock
-def test_preproc_filter(client):
+def test_where(client):
if sys.version_info >= (3, 10):
@meta(owner="nitin@fennel.ai")
@@ -91,3 +92,56 @@ def pipeline_window(cls, event: Dataset):
keys=pd.DataFrame({"user_id": [1, 2, 3, 4, 5]}),
)
assert df["age"].tolist() == [pd.NA, 1, pd.NA, pd.NA, pd.NA]
+
+
+@pytest.mark.integration
+@mock
+def test_where_expression(client):
+ if sys.version_info >= (3, 10):
+
+ @meta(owner="satwant@fennel.ai")
+ @source(
+ webhook.endpoint("A1"),
+ cdc="append",
+ disorder="14d",
+ env="prod",
+ where=eval(
+ (col("age") > 0) & (col("age") % 100 == 1), schema={"age": int}
+ ),
+ )
+ @dataset
+ class A1:
+ user_id: int
+ age: int
+ t: datetime
+
+ @dataset(index=True)
+ class A2:
+ user_id: int = field(key=True)
+ age: int
+ t: datetime
+
+ @pipeline
+ @inputs(A1)
+ def pipeline_window(cls, event: Dataset):
+ return event.groupby("user_id").latest()
+
+ client.commit(datasets=[A1, A2], message="first_commit", env="prod")
+
+ now = datetime.now(timezone.utc)
+ df = pd.DataFrame(
+ {
+ "user_id": [1, 1, 1, 2, 2, 3, 4, 5, 5, 5],
+ "age": [100, 11, 12, 1, 2, 2, 3, 3, 4, 5],
+ "t": [now, now, now, now, now, now, now, now, now, now],
+ }
+ )
+
+ client.log("fennel_webhook", "A1", df)
+ client.sleep(30)
+
+ df, _ = client.lookup(
+ A2,
+ keys=pd.DataFrame({"user_id": [1, 2, 3, 4, 5]}),
+ )
+ assert df["age"].tolist() == [pd.NA, 1, pd.NA, pd.NA, pd.NA]
diff --git a/fennel/connectors/__init__.py b/fennel/connectors/__init__.py
index cafdf31c3..e47c9ca7a 100644
--- a/fennel/connectors/__init__.py
+++ b/fennel/connectors/__init__.py
@@ -30,6 +30,7 @@
Ref,
ref,
PreProcValue,
+ WhereValue,
at_timestamp,
Eval,
eval,
diff --git a/fennel/connectors/connectors.py b/fennel/connectors/connectors.py
index 0d1ab77b1..a5c65c611 100644
--- a/fennel/connectors/connectors.py
+++ b/fennel/connectors/connectors.py
@@ -53,18 +53,19 @@ class Config:
arbitrary_types_allowed = True
-def ref(ref_name: str) -> PreProcValue:
+def ref(ref_name: str) -> Ref:
return Ref(name=ref_name)
def eval(
eval_type: Union[Callable, Expr, TypedExpr],
schema: Optional[Dict[str, Type]] = None,
-) -> PreProcValue:
+) -> Eval:
return Eval(eval_type=eval_type, additional_schema=schema)
PreProcValue = Union[Ref, Any, Eval]
+WhereValue = Union[Callable, Eval]
def preproc_has_indirection(preproc: Optional[Dict[str, PreProcValue]]):
@@ -120,7 +121,7 @@ def source(
preproc: Optional[Dict[str, PreProcValue]] = None,
bounded: bool = False,
idleness: Optional[Duration] = None,
- where: Optional[Callable] = None,
+ where: Optional[WhereValue] = None,
sample: Optional[Union[float, Sample]] = None,
) -> Callable[[T], Any]:
"""
@@ -815,7 +816,7 @@ class DataConnector:
pre_proc: Optional[Dict[str, PreProcValue]] = None
bounded: bool = False
idleness: Optional[Duration] = None
- where: Optional[Callable] = None
+ where: Optional[WhereValue] = None
sample: Optional[Union[float, Sample]] = None
how: Optional[Literal["incremental", "recreate"] | SnapshotData] = None
create: Optional[bool] = None
diff --git a/fennel/connectors/test_connectors.py b/fennel/connectors/test_connectors.py
index e36f42542..d4f008b86 100644
--- a/fennel/connectors/test_connectors.py
+++ b/fennel/connectors/test_connectors.py
@@ -4188,7 +4188,7 @@ def test_valid_preproc_value():
)
-def test_filter_preproc():
+def test_where():
if sys.version_info >= (3, 10):
@source(
@@ -4261,6 +4261,87 @@ class UserInfoDataset:
)
+def test_where_expression():
+ if sys.version_info >= (3, 10):
+
+ @source(
+ mysql.table(
+ "users",
+ cursor="added_on",
+ ),
+ every="1h",
+ disorder="20h",
+ bounded=True,
+ idleness="1h",
+ cdc="upsert",
+ where=eval(col("user_id") == 1, schema={"user_id": int}),
+ )
+ @meta(owner="test@test.com")
+ @dataset
+ class UserInfoDataset:
+ user_id: int = field(key=True)
+ name: str
+ gender: str
+ # Users date of birth
+ dob: str
+ age: int
+ account_creation_date: datetime
+ country: Optional[str]
+ timestamp: datetime = field(timestamp=True)
+
+ view = InternalTestClient()
+ view.add(UserInfoDataset)
+ sync_request = view._get_sync_request_proto()
+
+ assert len(sync_request.sources) == 1
+ source_request = sync_request.sources[0]
+ s = {
+ "table": {
+ "mysqlTable": {
+ "db": {
+ "name": "mysql",
+ "mysql": {
+ "host": "localhost",
+ "database": "test",
+ "user": "root",
+ "password": "root",
+ "port": 3306,
+ },
+ },
+ "tableName": "users",
+ }
+ },
+ "dataset": "UserInfoDataset",
+ "dsVersion": 1,
+ "every": "3600s",
+ "cursor": "added_on",
+ "disorder": "72000s",
+ "timestampField": "timestamp",
+ "cdc": "Upsert",
+ "bounded": True,
+ "idleness": "3600s",
+ "filter_schema": {
+ "fields": [{"name": "user_id", "dtype": {"intType": {}}}]
+ },
+ "filter_expr": {
+ "binary": {
+ "left": {"ref": {"name": "user_id"}},
+ "right": {
+ "jsonLiteral": {
+ "literal": "1",
+ "dtype": {"intType": {}},
+ }
+ },
+ "op": "EQ",
+ }
+ },
+ }
+ expected_source_request = ParseDict(s, connector_proto.Source())
+ assert source_request == expected_source_request, error_message(
+ source_request, expected_source_request
+ )
+
+
def test_assign_python_preproc():
if sys.version_info >= (3, 10):
diff --git a/fennel/connectors/test_invalid_connectors.py b/fennel/connectors/test_invalid_connectors.py
index 0533d74ad..b0037083c 100644
--- a/fennel/connectors/test_invalid_connectors.py
+++ b/fennel/connectors/test_invalid_connectors.py
@@ -1,5 +1,6 @@
from datetime import datetime, timezone
from lib2to3.fixes.fix_tuple_params import tuple_name
+import pandas as pd
from typing import Optional
from fennel.connectors.connectors import Protobuf, ref, eval
@@ -18,6 +19,7 @@
at_timestamp,
BigQuery,
S3Connector,
+ Webhook,
HTTP,
Certificate,
)
@@ -29,6 +31,10 @@
# noinspection PyUnresolvedReferences
from fennel.testing import *
+
+__owner__ = "test@test.com"
+webhook = Webhook(name="fennel_webhook")
+
mysql = MySQL(
name="mysql",
host="localhost",
@@ -1042,3 +1048,40 @@ class UserInfoDataset:
'`age` is of type `int` in Dataset `UserInfoDataset`, can not be cast to `float`. Full expression: `col("val1")`'
== str(e.value)
)
+
+
+@mock
+def test_invalid_where(client):
+
+ @source(
+ webhook.endpoint("UserInfoDataset"),
+ cdc="upsert",
+ disorder="14d",
+ env="prod",
+ where=eval(col("age") >= 18, schema={"age": float}),
+ )
+ @meta(owner="test@test.com")
+ @dataset
+ class UserInfoDataset:
+ user_id: int = field(key=True)
+ age: int
+ timestamp: datetime = field(timestamp=True)
+
+ client.commit(datasets=[UserInfoDataset], message="test")
+
+ with pytest.raises(Exception) as e:
+ now = datetime.now(timezone.utc)
+ df = pd.DataFrame(
+ {
+ "user_id": [1, 1, 1, 2, 2, 3, 4, 5, 5, 5],
+ "age": [100, 11, 12, 1, 2, 2, 3, 3, 4, 5],
+ "t": [now, now, now, now, now, now, now, now, now, now],
+ }
+ )
+ client.log("fennel_webhook", "UserInfoDataset", df)
+ assert (
+ "Schema validation failed during data insertion to `UserInfoDataset`: "
+ "Error using where for dataset `UserInfoDataset`: "
+ "Field `age` defined in schema for eval where has different type in the dataframe."
+ == str(e.value)
+ )
diff --git a/fennel/gen/connector_pb2.py b/fennel/gen/connector_pb2.py
index 8c191ccfd..19ef8a407 100644
--- a/fennel/gen/connector_pb2.py
+++ b/fennel/gen/connector_pb2.py
@@ -7,13 +7,18 @@
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
+
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
-from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2
-from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+from google.protobuf import (
+ duration_pb2 as google_dot_protobuf_dot_duration__pb2,
+)
+from google.protobuf import (
+ timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2,
+)
import fennel.gen.expression_pb2 as expression__pb2
import fennel.gen.kinesis_pb2 as kinesis__pb2
import fennel.gen.pycode_pb2 as pycode__pb2
@@ -22,121 +27,127 @@
import fennel.gen.secret_pb2 as secret__pb2
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x63onnector.proto\x12\x16\x66\x65nnel.proto.connector\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x10\x65xpression.proto\x1a\rkinesis.proto\x1a\x0cpycode.proto\x1a\x15schema_registry.proto\x1a\x0cschema.proto\x1a\x0csecret.proto\"\xba\x05\n\x0b\x45xtDatabase\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x05mysql\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.connector.MySQLH\x00\x12\x34\n\x08postgres\x18\x03 \x01(\x0b\x32 .fennel.proto.connector.PostgresH\x00\x12\x36\n\treference\x18\x04 \x01(\x0b\x32!.fennel.proto.connector.ReferenceH\x00\x12(\n\x02s3\x18\x05 \x01(\x0b\x32\x1a.fennel.proto.connector.S3H\x00\x12\x34\n\x08\x62igquery\x18\x06 \x01(\x0b\x32 .fennel.proto.connector.BigqueryH\x00\x12\x36\n\tsnowflake\x18\x07 \x01(\x0b\x32!.fennel.proto.connector.SnowflakeH\x00\x12.\n\x05kafka\x18\x08 \x01(\x0b\x32\x1d.fennel.proto.connector.KafkaH\x00\x12\x32\n\x07webhook\x18\t \x01(\x0b\x32\x1f.fennel.proto.connector.WebhookH\x00\x12\x32\n\x07kinesis\x18\n \x01(\x0b\x32\x1f.fennel.proto.connector.KinesisH\x00\x12\x34\n\x08redshift\x18\x0b \x01(\x0b\x32 .fennel.proto.connector.RedshiftH\x00\x12.\n\x05mongo\x18\x0c \x01(\x0b\x32\x1d.fennel.proto.connector.MongoH\x00\x12\x30\n\x06pubsub\x18\r \x01(\x0b\x32\x1e.fennel.proto.connector.PubSubH\x00\x12,\n\x04http\x18\x0e \x01(\x0b\x32\x1c.fennel.proto.connector.HttpH\x00\x42\t\n\x07variant\"?\n\x10SamplingStrategy\x12\x15\n\rsampling_rate\x18\x01 \x01(\x01\x12\x14\n\x0c\x63olumns_used\x18\x02 \x03(\t\"M\n\x0cPubSubFormat\x12\x32\n\x04json\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.JsonFormatH\x00\x42\t\n\x07variant\"\xbc\x01\n\x0bKafkaFormat\x12\x32\n\x04json\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.JsonFormatH\x00\x12\x32\n\x04\x61vro\x18\x02 \x01(\x0b\x32\".fennel.proto.connector.AvroFormatH\x00\x12:\n\x08protobuf\x18\x03 \x01(\x0b\x32&.fennel.proto.connector.ProtobufFormatH\x00\x42\t\n\x07variant\"\x10\n\x0e\x44\x65\x62\x65ziumFormat\"\x0c\n\nJsonFormat\"S\n\nAvroFormat\x12\x45\n\x0fschema_registry\x18\x01 \x01(\x0b\x32,.fennel.proto.schema_registry.SchemaRegistry\"W\n\x0eProtobufFormat\x12\x45\n\x0fschema_registry\x18\x01 \x01(\x0b\x32,.fennel.proto.schema_registry.SchemaRegistry\"\xde\x01\n\tReference\x12;\n\x06\x64\x62type\x18\x01 \x01(\x0e\x32+.fennel.proto.connector.Reference.ExtDBType\"\x93\x01\n\tExtDBType\x12\t\n\x05MYSQL\x10\x00\x12\x0c\n\x08POSTGRES\x10\x01\x12\x06\n\x02S3\x10\x02\x12\t\n\x05KAFKA\x10\x03\x12\x0c\n\x08\x42IGQUERY\x10\x04\x12\r\n\tSNOWFLAKE\x10\x05\x12\x0b\n\x07WEBHOOK\x10\x06\x12\x0b\n\x07KINESIS\x10\x07\x12\x0c\n\x08REDSHIFT\x10\x08\x12\t\n\x05MONGO\x10\t\x12\n\n\x06PUBSUB\x10\n\"E\n\x07Webhook\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\tretention\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\"\x8c\x02\n\x05MySQL\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x07 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\r\x12\x13\n\x0bjdbc_params\x18\x06 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant\"\x8f\x02\n\x08Postgres\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x07 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\r\x12\x13\n\x0bjdbc_params\x18\x06 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant\"\xb0\x02\n\x02S3\x12\x1f\n\x15\x61ws_secret_access_key\x18\x01 \x01(\tH\x00\x12\x46\n\x1c\x61ws_secret_access_key_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x1b\n\x11\x61ws_access_key_id\x18\x02 \x01(\tH\x01\x12\x42\n\x18\x61ws_access_key_id_secret\x18\x05 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x15\n\x08role_arn\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x1f\n\x1d\x61ws_secret_access_key_variantB\x1b\n\x19\x61ws_access_key_id_variantB\x0b\n\t_role_arn\"\xb6\x01\n\x08\x42igquery\x12\x1d\n\x13service_account_key\x18\x02 \x01(\tH\x00\x12\x44\n\x1aservice_account_key_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\ndataset_id\x18\x01 \x01(\t\x12\x12\n\nproject_id\x18\x03 \x01(\tB\x1d\n\x1bservice_account_key_variant\"\xa1\x02\n\tSnowflake\x12\x0e\n\x04user\x18\x02 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x03 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\t \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0f\n\x07\x61\x63\x63ount\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x04 \x01(\t\x12\x11\n\twarehouse\x18\x05 \x01(\t\x12\x0c\n\x04role\x18\x06 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x07 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant\"\xf9\x02\n\x05Kafka\x12\x1d\n\x13sasl_plain_username\x18\x05 \x01(\tH\x00\x12>\n\x14sasl_username_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x1d\n\x13sasl_plain_password\x18\x06 \x01(\tH\x01\x12>\n\x14sasl_password_secret\x18\t \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x19\n\x11\x62ootstrap_servers\x18\x01 \x01(\t\x12\x19\n\x11security_protocol\x18\x02 \x01(\t\x12\x16\n\x0esasl_mechanism\x18\x03 \x01(\t\x12\x1c\n\x10sasl_jaas_config\x18\x04 \x01(\tB\x02\x18\x01\x12\x14\n\x08group_id\x18\x07 \x01(\tB\x02\x18\x01\x42\x17\n\x15sasl_username_variantB\x17\n\x15sasl_password_variant\"\x1b\n\x07Kinesis\x12\x10\n\x08role_arn\x18\x01 \x01(\t\"\xd3\x01\n\x0b\x43redentials\x12\x12\n\x08username\x18\x01 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x03 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x02 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x42\x12\n\x10username_variantB\x12\n\x10password_variant\"}\n\x16RedshiftAuthentication\x12\x1c\n\x12s3_access_role_arn\x18\x01 \x01(\tH\x00\x12:\n\x0b\x63redentials\x18\x02 \x01(\x0b\x32#.fennel.proto.connector.CredentialsH\x00\x42\t\n\x07variant\"\x99\x01\n\x08Redshift\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x03 \x01(\r\x12\x0e\n\x06schema\x18\x04 \x01(\t\x12O\n\x17redshift_authentication\x18\x05 \x01(\x0b\x32..fennel.proto.connector.RedshiftAuthentication\"\xe9\x01\n\x05Mongo\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x05 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x06 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant\"\xa0\x01\n\x06PubSub\x12\x1d\n\x13service_account_key\x18\x02 \x01(\tH\x00\x12\x44\n\x1aservice_account_key_secret\x18\x03 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\nproject_id\x18\x01 \x01(\tB\x1d\n\x1bservice_account_key_variant\"s\n\x0eSensitiveDatum\x12\x10\n\x06secret\x18\x01 \x01(\tH\x00\x12\x34\n\nsecret_ref\x18\x02 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x42\x19\n\x17sensitive_datum_variant\"\xb8\x01\n\x04Http\x12\x0e\n\x04host\x18\x01 \x01(\tH\x00\x12\x35\n\x0bhost_secret\x18\x02 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x0f\n\x07healthz\x18\x03 \x01(\t\x12<\n\x07\x63\x61_cert\x18\x04 \x01(\x0b\x32&.fennel.proto.connector.SensitiveDatumH\x01\x88\x01\x01\x42\x0e\n\x0chost_variantB\n\n\x08_ca_cert\"\xf7\x05\n\x08\x45xtTable\x12\x39\n\x0bmysql_table\x18\x01 \x01(\x0b\x32\".fennel.proto.connector.MySQLTableH\x00\x12\x39\n\x08pg_table\x18\x02 \x01(\x0b\x32%.fennel.proto.connector.PostgresTableH\x00\x12\x33\n\x08s3_table\x18\x03 \x01(\x0b\x32\x1f.fennel.proto.connector.S3TableH\x00\x12\x39\n\x0bkafka_topic\x18\x04 \x01(\x0b\x32\".fennel.proto.connector.KafkaTopicH\x00\x12\x41\n\x0fsnowflake_table\x18\x05 \x01(\x0b\x32&.fennel.proto.connector.SnowflakeTableH\x00\x12?\n\x0e\x62igquery_table\x18\x06 \x01(\x0b\x32%.fennel.proto.connector.BigqueryTableH\x00\x12;\n\x08\x65ndpoint\x18\x07 \x01(\x0b\x32\'.fennel.proto.connector.WebhookEndpointH\x00\x12?\n\x0ekinesis_stream\x18\x08 \x01(\x0b\x32%.fennel.proto.connector.KinesisStreamH\x00\x12?\n\x0eredshift_table\x18\t \x01(\x0b\x32%.fennel.proto.connector.RedshiftTableH\x00\x12\x43\n\x10mongo_collection\x18\n \x01(\x0b\x32\'.fennel.proto.connector.MongoCollectionH\x00\x12;\n\x0cpubsub_topic\x18\x0b \x01(\x0b\x32#.fennel.proto.connector.PubSubTopicH\x00\x12\x35\n\thttp_path\x18\x0c \x01(\x0b\x32 .fennel.proto.connector.HttpPathH\x00\x42\t\n\x07variant\"Q\n\nMySQLTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"z\n\rPostgresTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12\x16\n\tslot_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x42\x0c\n\n_slot_name\"\xf7\x01\n\x07S3Table\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x13\n\x0bpath_prefix\x18\x02 \x01(\t\x12\x11\n\tdelimiter\x18\x04 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x05 \x01(\t\x12/\n\x02\x64\x62\x18\x06 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\npre_sorted\x18\x07 \x01(\x08\x12\x13\n\x0bpath_suffix\x18\x08 \x01(\t\x12.\n\x06spread\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x88\x01\x01\x12\x0f\n\x07headers\x18\t \x03(\tB\t\n\x07_spread\"\x81\x01\n\nKafkaTopic\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x33\n\x06\x66ormat\x18\x03 \x01(\x0b\x32#.fennel.proto.connector.KafkaFormat\"T\n\rBigqueryTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"U\n\x0eSnowflakeTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"\x81\x01\n\x0fWebhookEndpoint\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12+\n\x08\x64uration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\"\xd3\x01\n\rKinesisStream\x12\x12\n\nstream_arn\x18\x01 \x01(\t\x12\x39\n\rinit_position\x18\x02 \x01(\x0e\x32\".fennel.proto.kinesis.InitPosition\x12\x32\n\x0einit_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x66ormat\x18\x04 \x01(\t\x12/\n\x02\x64\x62\x18\x05 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\"T\n\rRedshiftTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\"\x86\x01\n\x0bPubSubTopic\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08topic_id\x18\x02 \x01(\t\x12\x34\n\x06\x66ormat\x18\x03 \x01(\x0b\x32$.fennel.proto.connector.PubSubFormat\"\xdb\x01\n\x08HttpPath\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x12\n\x05limit\x18\x03 \x01(\rH\x00\x88\x01\x01\x12>\n\x07headers\x18\x04 \x03(\x0b\x32-.fennel.proto.connector.HttpPath.HeadersEntry\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x08\n\x06_limit\"\xb1\x02\n\x0cPreProcValue\x12\r\n\x03ref\x18\x01 \x01(\tH\x00\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1a.fennel.proto.schema.ValueH\x00\x12\x39\n\x04\x65val\x18\x03 \x01(\x0b\x32).fennel.proto.connector.PreProcValue.EvalH\x00\x1a\x9e\x01\n\x04\x45val\x12+\n\x06schema\x18\x01 \x01(\x0b\x32\x1b.fennel.proto.schema.Schema\x12-\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.ExprH\x00\x12-\n\x06pycode\x18\x03 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCodeH\x00\x42\x0b\n\teval_typeB\t\n\x07variant\"[\n\x0fMongoCollection\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x17\n\x0f\x63ollection_name\x18\x02 \x01(\t\"2\n\x0cSnapshotData\x12\x0e\n\x06marker\x18\x01 \x01(\t\x12\x12\n\nnum_retain\x18\x02 \x01(\r\"\r\n\x0bIncremental\"\n\n\x08Recreate\"\xbc\x01\n\x05Style\x12:\n\x0bincremental\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.IncrementalH\x00\x12\x34\n\x08recreate\x18\x02 \x01(\x0b\x32 .fennel.proto.connector.RecreateH\x00\x12\x38\n\x08snapshot\x18\x03 \x01(\x0b\x32$.fennel.proto.connector.SnapshotDataH\x00\x42\x07\n\x05Style\"\x91\x06\n\x06Source\x12/\n\x05table\x18\x01 \x01(\x0b\x32 .fennel.proto.connector.ExtTable\x12\x0f\n\x07\x64\x61taset\x18\x02 \x01(\t\x12\x12\n\nds_version\x18\x03 \x01(\r\x12(\n\x05\x65very\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x06\x63ursor\x18\x05 \x01(\tH\x00\x88\x01\x01\x12+\n\x08\x64isorder\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x17\n\x0ftimestamp_field\x18\x07 \x01(\t\x12\x30\n\x03\x63\x64\x63\x18\x08 \x01(\x0e\x32#.fennel.proto.connector.CDCStrategy\x12\x31\n\rstarting_from\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12=\n\x08pre_proc\x18\n \x03(\x0b\x32+.fennel.proto.connector.Source.PreProcEntry\x12\x0f\n\x07version\x18\x0b \x01(\r\x12\x0f\n\x07\x62ounded\x18\x0c \x01(\x08\x12\x30\n\x08idleness\x18\r \x01(\x0b\x32\x19.google.protobuf.DurationH\x01\x88\x01\x01\x12)\n\x05until\x18\x0e \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x06\x66ilter\x18\x0f \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCodeH\x02\x88\x01\x01\x12H\n\x11sampling_strategy\x18\x10 \x01(\x0b\x32(.fennel.proto.connector.SamplingStrategyH\x03\x88\x01\x01\x1aT\n\x0cPreProcEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.fennel.proto.connector.PreProcValue:\x02\x38\x01\x42\t\n\x07_cursorB\x0b\n\t_idlenessB\t\n\x07_filterB\x14\n\x12_sampling_strategy\"\xee\x03\n\x04Sink\x12/\n\x05table\x18\x01 \x01(\x0b\x32 .fennel.proto.connector.ExtTable\x12\x0f\n\x07\x64\x61taset\x18\x02 \x01(\t\x12\x12\n\nds_version\x18\x03 \x01(\r\x12\x35\n\x03\x63\x64\x63\x18\x04 \x01(\x0e\x32#.fennel.proto.connector.CDCStrategyH\x00\x88\x01\x01\x12(\n\x05\x65very\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x03how\x18\x06 \x01(\x0b\x32\x1d.fennel.proto.connector.StyleH\x01\x88\x01\x01\x12\x0e\n\x06\x63reate\x18\x07 \x01(\x08\x12:\n\x07renames\x18\x08 \x03(\x0b\x32).fennel.proto.connector.Sink.RenamesEntry\x12.\n\x05since\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x02\x88\x01\x01\x12.\n\x05until\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x03\x88\x01\x01\x1a.\n\x0cRenamesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x06\n\x04_cdcB\x06\n\x04_howB\x08\n\x06_sinceB\x08\n\x06_until*K\n\x0b\x43\x44\x43Strategy\x12\n\n\x06\x41ppend\x10\x00\x12\n\n\x06Upsert\x10\x01\x12\x0c\n\x08\x44\x65\x62\x65zium\x10\x02\x12\n\n\x06Native\x10\x03\x12\n\n\x06\x44\x65lete\x10\x04\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
+ b'\n\x0f\x63onnector.proto\x12\x16\x66\x65nnel.proto.connector\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x10\x65xpression.proto\x1a\rkinesis.proto\x1a\x0cpycode.proto\x1a\x15schema_registry.proto\x1a\x0cschema.proto\x1a\x0csecret.proto"\xba\x05\n\x0b\x45xtDatabase\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x05mysql\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.connector.MySQLH\x00\x12\x34\n\x08postgres\x18\x03 \x01(\x0b\x32 .fennel.proto.connector.PostgresH\x00\x12\x36\n\treference\x18\x04 \x01(\x0b\x32!.fennel.proto.connector.ReferenceH\x00\x12(\n\x02s3\x18\x05 \x01(\x0b\x32\x1a.fennel.proto.connector.S3H\x00\x12\x34\n\x08\x62igquery\x18\x06 \x01(\x0b\x32 .fennel.proto.connector.BigqueryH\x00\x12\x36\n\tsnowflake\x18\x07 \x01(\x0b\x32!.fennel.proto.connector.SnowflakeH\x00\x12.\n\x05kafka\x18\x08 \x01(\x0b\x32\x1d.fennel.proto.connector.KafkaH\x00\x12\x32\n\x07webhook\x18\t \x01(\x0b\x32\x1f.fennel.proto.connector.WebhookH\x00\x12\x32\n\x07kinesis\x18\n \x01(\x0b\x32\x1f.fennel.proto.connector.KinesisH\x00\x12\x34\n\x08redshift\x18\x0b \x01(\x0b\x32 .fennel.proto.connector.RedshiftH\x00\x12.\n\x05mongo\x18\x0c \x01(\x0b\x32\x1d.fennel.proto.connector.MongoH\x00\x12\x30\n\x06pubsub\x18\r \x01(\x0b\x32\x1e.fennel.proto.connector.PubSubH\x00\x12,\n\x04http\x18\x0e \x01(\x0b\x32\x1c.fennel.proto.connector.HttpH\x00\x42\t\n\x07variant"?\n\x10SamplingStrategy\x12\x15\n\rsampling_rate\x18\x01 \x01(\x01\x12\x14\n\x0c\x63olumns_used\x18\x02 \x03(\t"M\n\x0cPubSubFormat\x12\x32\n\x04json\x18\x01 \x01(\x0b\x32".fennel.proto.connector.JsonFormatH\x00\x42\t\n\x07variant"\xbc\x01\n\x0bKafkaFormat\x12\x32\n\x04json\x18\x01 \x01(\x0b\x32".fennel.proto.connector.JsonFormatH\x00\x12\x32\n\x04\x61vro\x18\x02 \x01(\x0b\x32".fennel.proto.connector.AvroFormatH\x00\x12:\n\x08protobuf\x18\x03 \x01(\x0b\x32&.fennel.proto.connector.ProtobufFormatH\x00\x42\t\n\x07variant"\x10\n\x0e\x44\x65\x62\x65ziumFormat"\x0c\n\nJsonFormat"S\n\nAvroFormat\x12\x45\n\x0fschema_registry\x18\x01 \x01(\x0b\x32,.fennel.proto.schema_registry.SchemaRegistry"W\n\x0eProtobufFormat\x12\x45\n\x0fschema_registry\x18\x01 \x01(\x0b\x32,.fennel.proto.schema_registry.SchemaRegistry"\xde\x01\n\tReference\x12;\n\x06\x64\x62type\x18\x01 \x01(\x0e\x32+.fennel.proto.connector.Reference.ExtDBType"\x93\x01\n\tExtDBType\x12\t\n\x05MYSQL\x10\x00\x12\x0c\n\x08POSTGRES\x10\x01\x12\x06\n\x02S3\x10\x02\x12\t\n\x05KAFKA\x10\x03\x12\x0c\n\x08\x42IGQUERY\x10\x04\x12\r\n\tSNOWFLAKE\x10\x05\x12\x0b\n\x07WEBHOOK\x10\x06\x12\x0b\n\x07KINESIS\x10\x07\x12\x0c\n\x08REDSHIFT\x10\x08\x12\t\n\x05MONGO\x10\t\x12\n\n\x06PUBSUB\x10\n"E\n\x07Webhook\x12\x0c\n\x04name\x18\x01 \x01(\t\x12,\n\tretention\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x8c\x02\n\x05MySQL\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x07 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\r\x12\x13\n\x0bjdbc_params\x18\x06 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant"\x8f\x02\n\x08Postgres\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x07 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\r\x12\x13\n\x0bjdbc_params\x18\x06 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant"\xb0\x02\n\x02S3\x12\x1f\n\x15\x61ws_secret_access_key\x18\x01 \x01(\tH\x00\x12\x46\n\x1c\x61ws_secret_access_key_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x1b\n\x11\x61ws_access_key_id\x18\x02 \x01(\tH\x01\x12\x42\n\x18\x61ws_access_key_id_secret\x18\x05 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x15\n\x08role_arn\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x1f\n\x1d\x61ws_secret_access_key_variantB\x1b\n\x19\x61ws_access_key_id_variantB\x0b\n\t_role_arn"\xb6\x01\n\x08\x42igquery\x12\x1d\n\x13service_account_key\x18\x02 \x01(\tH\x00\x12\x44\n\x1aservice_account_key_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\ndataset_id\x18\x01 \x01(\t\x12\x12\n\nproject_id\x18\x03 \x01(\tB\x1d\n\x1bservice_account_key_variant"\xa1\x02\n\tSnowflake\x12\x0e\n\x04user\x18\x02 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x03 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\t \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0f\n\x07\x61\x63\x63ount\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x04 \x01(\t\x12\x11\n\twarehouse\x18\x05 \x01(\t\x12\x0c\n\x04role\x18\x06 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x07 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant"\xf9\x02\n\x05Kafka\x12\x1d\n\x13sasl_plain_username\x18\x05 \x01(\tH\x00\x12>\n\x14sasl_username_secret\x18\x08 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x1d\n\x13sasl_plain_password\x18\x06 \x01(\tH\x01\x12>\n\x14sasl_password_secret\x18\t \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x19\n\x11\x62ootstrap_servers\x18\x01 \x01(\t\x12\x19\n\x11security_protocol\x18\x02 \x01(\t\x12\x16\n\x0esasl_mechanism\x18\x03 \x01(\t\x12\x1c\n\x10sasl_jaas_config\x18\x04 \x01(\tB\x02\x18\x01\x12\x14\n\x08group_id\x18\x07 \x01(\tB\x02\x18\x01\x42\x17\n\x15sasl_username_variantB\x17\n\x15sasl_password_variant"\x1b\n\x07Kinesis\x12\x10\n\x08role_arn\x18\x01 \x01(\t"\xd3\x01\n\x0b\x43redentials\x12\x12\n\x08username\x18\x01 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x03 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x02 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x04 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x42\x12\n\x10username_variantB\x12\n\x10password_variant"}\n\x16RedshiftAuthentication\x12\x1c\n\x12s3_access_role_arn\x18\x01 \x01(\tH\x00\x12:\n\x0b\x63redentials\x18\x02 \x01(\x0b\x32#.fennel.proto.connector.CredentialsH\x00\x42\t\n\x07variant"\x99\x01\n\x08Redshift\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\x0c\n\x04port\x18\x03 \x01(\r\x12\x0e\n\x06schema\x18\x04 \x01(\t\x12O\n\x17redshift_authentication\x18\x05 \x01(\x0b\x32..fennel.proto.connector.RedshiftAuthentication"\xe9\x01\n\x05Mongo\x12\x0e\n\x04user\x18\x03 \x01(\tH\x00\x12\x39\n\x0fusername_secret\x18\x05 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\x08password\x18\x04 \x01(\tH\x01\x12\x39\n\x0fpassword_secret\x18\x06 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x01\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\tB\x12\n\x10username_variantB\x12\n\x10password_variant"\xa0\x01\n\x06PubSub\x12\x1d\n\x13service_account_key\x18\x02 \x01(\tH\x00\x12\x44\n\x1aservice_account_key_secret\x18\x03 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x12\n\nproject_id\x18\x01 \x01(\tB\x1d\n\x1bservice_account_key_variant"s\n\x0eSensitiveDatum\x12\x10\n\x06secret\x18\x01 \x01(\tH\x00\x12\x34\n\nsecret_ref\x18\x02 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x42\x19\n\x17sensitive_datum_variant"\xb8\x01\n\x04Http\x12\x0e\n\x04host\x18\x01 \x01(\tH\x00\x12\x35\n\x0bhost_secret\x18\x02 \x01(\x0b\x32\x1e.fennel.proto.secret.SecretRefH\x00\x12\x0f\n\x07healthz\x18\x03 \x01(\t\x12<\n\x07\x63\x61_cert\x18\x04 \x01(\x0b\x32&.fennel.proto.connector.SensitiveDatumH\x01\x88\x01\x01\x42\x0e\n\x0chost_variantB\n\n\x08_ca_cert"\xf7\x05\n\x08\x45xtTable\x12\x39\n\x0bmysql_table\x18\x01 \x01(\x0b\x32".fennel.proto.connector.MySQLTableH\x00\x12\x39\n\x08pg_table\x18\x02 \x01(\x0b\x32%.fennel.proto.connector.PostgresTableH\x00\x12\x33\n\x08s3_table\x18\x03 \x01(\x0b\x32\x1f.fennel.proto.connector.S3TableH\x00\x12\x39\n\x0bkafka_topic\x18\x04 \x01(\x0b\x32".fennel.proto.connector.KafkaTopicH\x00\x12\x41\n\x0fsnowflake_table\x18\x05 \x01(\x0b\x32&.fennel.proto.connector.SnowflakeTableH\x00\x12?\n\x0e\x62igquery_table\x18\x06 \x01(\x0b\x32%.fennel.proto.connector.BigqueryTableH\x00\x12;\n\x08\x65ndpoint\x18\x07 \x01(\x0b\x32\'.fennel.proto.connector.WebhookEndpointH\x00\x12?\n\x0ekinesis_stream\x18\x08 \x01(\x0b\x32%.fennel.proto.connector.KinesisStreamH\x00\x12?\n\x0eredshift_table\x18\t \x01(\x0b\x32%.fennel.proto.connector.RedshiftTableH\x00\x12\x43\n\x10mongo_collection\x18\n \x01(\x0b\x32\'.fennel.proto.connector.MongoCollectionH\x00\x12;\n\x0cpubsub_topic\x18\x0b \x01(\x0b\x32#.fennel.proto.connector.PubSubTopicH\x00\x12\x35\n\thttp_path\x18\x0c \x01(\x0b\x32 .fennel.proto.connector.HttpPathH\x00\x42\t\n\x07variant"Q\n\nMySQLTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t"z\n\rPostgresTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12\x16\n\tslot_name\x18\x03 \x01(\tH\x00\x88\x01\x01\x42\x0c\n\n_slot_name"\xf7\x01\n\x07S3Table\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\x12\x13\n\x0bpath_prefix\x18\x02 \x01(\t\x12\x11\n\tdelimiter\x18\x04 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x05 \x01(\t\x12/\n\x02\x64\x62\x18\x06 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\npre_sorted\x18\x07 \x01(\x08\x12\x13\n\x0bpath_suffix\x18\x08 \x01(\t\x12.\n\x06spread\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x88\x01\x01\x12\x0f\n\x07headers\x18\t \x03(\tB\t\n\x07_spread"\x81\x01\n\nKafkaTopic\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x33\n\x06\x66ormat\x18\x03 \x01(\x0b\x32#.fennel.proto.connector.KafkaFormat"T\n\rBigqueryTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t"U\n\x0eSnowflakeTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t"\x81\x01\n\x0fWebhookEndpoint\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12+\n\x08\x64uration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration"\xd3\x01\n\rKinesisStream\x12\x12\n\nstream_arn\x18\x01 \x01(\t\x12\x39\n\rinit_position\x18\x02 \x01(\x0e\x32".fennel.proto.kinesis.InitPosition\x12\x32\n\x0einit_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x66ormat\x18\x04 \x01(\t\x12/\n\x02\x64\x62\x18\x05 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase"T\n\rRedshiftTable\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x12\n\ntable_name\x18\x02 \x01(\t"\x86\x01\n\x0bPubSubTopic\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08topic_id\x18\x02 \x01(\t\x12\x34\n\x06\x66ormat\x18\x03 \x01(\x0b\x32$.fennel.proto.connector.PubSubFormat"\xdb\x01\n\x08HttpPath\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x12\n\x05limit\x18\x03 \x01(\rH\x00\x88\x01\x01\x12>\n\x07headers\x18\x04 \x03(\x0b\x32-.fennel.proto.connector.HttpPath.HeadersEntry\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x08\n\x06_limit"\x9e\x01\n\x04\x45val\x12+\n\x06schema\x18\x01 \x01(\x0b\x32\x1b.fennel.proto.schema.Schema\x12-\n\x04\x65xpr\x18\x02 \x01(\x0b\x32\x1d.fennel.proto.expression.ExprH\x00\x12-\n\x06pycode\x18\x03 \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCodeH\x00\x42\x0b\n\teval_type"\x83\x01\n\x0cPreProcValue\x12\r\n\x03ref\x18\x01 \x01(\tH\x00\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1a.fennel.proto.schema.ValueH\x00\x12,\n\x04\x65val\x18\x03 \x01(\x0b\x32\x1c.fennel.proto.connector.EvalH\x00\x42\t\n\x07variant"[\n\x0fMongoCollection\x12/\n\x02\x64\x62\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.ExtDatabase\x12\x17\n\x0f\x63ollection_name\x18\x02 \x01(\t"2\n\x0cSnapshotData\x12\x0e\n\x06marker\x18\x01 \x01(\t\x12\x12\n\nnum_retain\x18\x02 \x01(\r"\r\n\x0bIncremental"\n\n\x08Recreate"\xbc\x01\n\x05Style\x12:\n\x0bincremental\x18\x01 \x01(\x0b\x32#.fennel.proto.connector.IncrementalH\x00\x12\x34\n\x08recreate\x18\x02 \x01(\x0b\x32 .fennel.proto.connector.RecreateH\x00\x12\x38\n\x08snapshot\x18\x03 \x01(\x0b\x32$.fennel.proto.connector.SnapshotDataH\x00\x42\x07\n\x05Style"\xa5\x07\n\x06Source\x12/\n\x05table\x18\x01 \x01(\x0b\x32 .fennel.proto.connector.ExtTable\x12\x0f\n\x07\x64\x61taset\x18\x02 \x01(\t\x12\x12\n\nds_version\x18\x03 \x01(\r\x12(\n\x05\x65very\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x06\x63ursor\x18\x05 \x01(\tH\x00\x88\x01\x01\x12+\n\x08\x64isorder\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x17\n\x0ftimestamp_field\x18\x07 \x01(\t\x12\x30\n\x03\x63\x64\x63\x18\x08 \x01(\x0e\x32#.fennel.proto.connector.CDCStrategy\x12\x31\n\rstarting_from\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12=\n\x08pre_proc\x18\n \x03(\x0b\x32+.fennel.proto.connector.Source.PreProcEntry\x12\x0f\n\x07version\x18\x0b \x01(\r\x12\x0f\n\x07\x62ounded\x18\x0c \x01(\x08\x12\x30\n\x08idleness\x18\r \x01(\x0b\x32\x19.google.protobuf.DurationH\x01\x88\x01\x01\x12)\n\x05until\x18\x0e \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x06\x66ilter\x18\x0f \x01(\x0b\x32\x1b.fennel.proto.pycode.PyCodeH\x02\x88\x01\x01\x12H\n\x11sampling_strategy\x18\x10 \x01(\x0b\x32(.fennel.proto.connector.SamplingStrategyH\x03\x88\x01\x01\x12\x37\n\x0b\x66ilter_expr\x18\x11 \x01(\x0b\x32\x1d.fennel.proto.expression.ExprH\x04\x88\x01\x01\x12\x37\n\rfilter_schema\x18\x12 \x01(\x0b\x32\x1b.fennel.proto.schema.SchemaH\x05\x88\x01\x01\x1aT\n\x0cPreProcEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.fennel.proto.connector.PreProcValue:\x02\x38\x01\x42\t\n\x07_cursorB\x0b\n\t_idlenessB\t\n\x07_filterB\x14\n\x12_sampling_strategyB\x0e\n\x0c_filter_exprB\x10\n\x0e_filter_schema"\xee\x03\n\x04Sink\x12/\n\x05table\x18\x01 \x01(\x0b\x32 .fennel.proto.connector.ExtTable\x12\x0f\n\x07\x64\x61taset\x18\x02 \x01(\t\x12\x12\n\nds_version\x18\x03 \x01(\r\x12\x35\n\x03\x63\x64\x63\x18\x04 \x01(\x0e\x32#.fennel.proto.connector.CDCStrategyH\x00\x88\x01\x01\x12(\n\x05\x65very\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x03how\x18\x06 \x01(\x0b\x32\x1d.fennel.proto.connector.StyleH\x01\x88\x01\x01\x12\x0e\n\x06\x63reate\x18\x07 \x01(\x08\x12:\n\x07renames\x18\x08 \x03(\x0b\x32).fennel.proto.connector.Sink.RenamesEntry\x12.\n\x05since\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x02\x88\x01\x01\x12.\n\x05until\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x03\x88\x01\x01\x1a.\n\x0cRenamesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x06\n\x04_cdcB\x06\n\x04_howB\x08\n\x06_sinceB\x08\n\x06_until*K\n\x0b\x43\x44\x43Strategy\x12\n\n\x06\x41ppend\x10\x00\x12\n\n\x06Upsert\x10\x01\x12\x0c\n\x08\x44\x65\x62\x65zium\x10\x02\x12\n\n\x06Native\x10\x03\x12\n\n\x06\x44\x65lete\x10\x04\x62\x06proto3'
+)
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'connector_pb2', _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "connector_pb2", _globals)
if not _descriptor._USE_C_DESCRIPTORS:
- DESCRIPTOR._loaded_options = None
- _globals['_KAFKA'].fields_by_name['sasl_jaas_config']._loaded_options = None
- _globals['_KAFKA'].fields_by_name['sasl_jaas_config']._serialized_options = b'\030\001'
- _globals['_KAFKA'].fields_by_name['group_id']._loaded_options = None
- _globals['_KAFKA'].fields_by_name['group_id']._serialized_options = b'\030\001'
- _globals['_HTTPPATH_HEADERSENTRY']._loaded_options = None
- _globals['_HTTPPATH_HEADERSENTRY']._serialized_options = b'8\001'
- _globals['_SOURCE_PREPROCENTRY']._loaded_options = None
- _globals['_SOURCE_PREPROCENTRY']._serialized_options = b'8\001'
- _globals['_SINK_RENAMESENTRY']._loaded_options = None
- _globals['_SINK_RENAMESENTRY']._serialized_options = b'8\001'
- _globals['_CDCSTRATEGY']._serialized_start=8953
- _globals['_CDCSTRATEGY']._serialized_end=9028
- _globals['_EXTDATABASE']._serialized_start=207
- _globals['_EXTDATABASE']._serialized_end=905
- _globals['_SAMPLINGSTRATEGY']._serialized_start=907
- _globals['_SAMPLINGSTRATEGY']._serialized_end=970
- _globals['_PUBSUBFORMAT']._serialized_start=972
- _globals['_PUBSUBFORMAT']._serialized_end=1049
- _globals['_KAFKAFORMAT']._serialized_start=1052
- _globals['_KAFKAFORMAT']._serialized_end=1240
- _globals['_DEBEZIUMFORMAT']._serialized_start=1242
- _globals['_DEBEZIUMFORMAT']._serialized_end=1258
- _globals['_JSONFORMAT']._serialized_start=1260
- _globals['_JSONFORMAT']._serialized_end=1272
- _globals['_AVROFORMAT']._serialized_start=1274
- _globals['_AVROFORMAT']._serialized_end=1357
- _globals['_PROTOBUFFORMAT']._serialized_start=1359
- _globals['_PROTOBUFFORMAT']._serialized_end=1446
- _globals['_REFERENCE']._serialized_start=1449
- _globals['_REFERENCE']._serialized_end=1671
- _globals['_REFERENCE_EXTDBTYPE']._serialized_start=1524
- _globals['_REFERENCE_EXTDBTYPE']._serialized_end=1671
- _globals['_WEBHOOK']._serialized_start=1673
- _globals['_WEBHOOK']._serialized_end=1742
- _globals['_MYSQL']._serialized_start=1745
- _globals['_MYSQL']._serialized_end=2013
- _globals['_POSTGRES']._serialized_start=2016
- _globals['_POSTGRES']._serialized_end=2287
- _globals['_S3']._serialized_start=2290
- _globals['_S3']._serialized_end=2594
- _globals['_BIGQUERY']._serialized_start=2597
- _globals['_BIGQUERY']._serialized_end=2779
- _globals['_SNOWFLAKE']._serialized_start=2782
- _globals['_SNOWFLAKE']._serialized_end=3071
- _globals['_KAFKA']._serialized_start=3074
- _globals['_KAFKA']._serialized_end=3451
- _globals['_KINESIS']._serialized_start=3453
- _globals['_KINESIS']._serialized_end=3480
- _globals['_CREDENTIALS']._serialized_start=3483
- _globals['_CREDENTIALS']._serialized_end=3694
- _globals['_REDSHIFTAUTHENTICATION']._serialized_start=3696
- _globals['_REDSHIFTAUTHENTICATION']._serialized_end=3821
- _globals['_REDSHIFT']._serialized_start=3824
- _globals['_REDSHIFT']._serialized_end=3977
- _globals['_MONGO']._serialized_start=3980
- _globals['_MONGO']._serialized_end=4213
- _globals['_PUBSUB']._serialized_start=4216
- _globals['_PUBSUB']._serialized_end=4376
- _globals['_SENSITIVEDATUM']._serialized_start=4378
- _globals['_SENSITIVEDATUM']._serialized_end=4493
- _globals['_HTTP']._serialized_start=4496
- _globals['_HTTP']._serialized_end=4680
- _globals['_EXTTABLE']._serialized_start=4683
- _globals['_EXTTABLE']._serialized_end=5442
- _globals['_MYSQLTABLE']._serialized_start=5444
- _globals['_MYSQLTABLE']._serialized_end=5525
- _globals['_POSTGRESTABLE']._serialized_start=5527
- _globals['_POSTGRESTABLE']._serialized_end=5649
- _globals['_S3TABLE']._serialized_start=5652
- _globals['_S3TABLE']._serialized_end=5899
- _globals['_KAFKATOPIC']._serialized_start=5902
- _globals['_KAFKATOPIC']._serialized_end=6031
- _globals['_BIGQUERYTABLE']._serialized_start=6033
- _globals['_BIGQUERYTABLE']._serialized_end=6117
- _globals['_SNOWFLAKETABLE']._serialized_start=6119
- _globals['_SNOWFLAKETABLE']._serialized_end=6204
- _globals['_WEBHOOKENDPOINT']._serialized_start=6207
- _globals['_WEBHOOKENDPOINT']._serialized_end=6336
- _globals['_KINESISSTREAM']._serialized_start=6339
- _globals['_KINESISSTREAM']._serialized_end=6550
- _globals['_REDSHIFTTABLE']._serialized_start=6552
- _globals['_REDSHIFTTABLE']._serialized_end=6636
- _globals['_PUBSUBTOPIC']._serialized_start=6639
- _globals['_PUBSUBTOPIC']._serialized_end=6773
- _globals['_HTTPPATH']._serialized_start=6776
- _globals['_HTTPPATH']._serialized_end=6995
- _globals['_HTTPPATH_HEADERSENTRY']._serialized_start=6939
- _globals['_HTTPPATH_HEADERSENTRY']._serialized_end=6985
- _globals['_PREPROCVALUE']._serialized_start=6998
- _globals['_PREPROCVALUE']._serialized_end=7303
- _globals['_PREPROCVALUE_EVAL']._serialized_start=7134
- _globals['_PREPROCVALUE_EVAL']._serialized_end=7292
- _globals['_MONGOCOLLECTION']._serialized_start=7305
- _globals['_MONGOCOLLECTION']._serialized_end=7396
- _globals['_SNAPSHOTDATA']._serialized_start=7398
- _globals['_SNAPSHOTDATA']._serialized_end=7448
- _globals['_INCREMENTAL']._serialized_start=7450
- _globals['_INCREMENTAL']._serialized_end=7463
- _globals['_RECREATE']._serialized_start=7465
- _globals['_RECREATE']._serialized_end=7475
- _globals['_STYLE']._serialized_start=7478
- _globals['_STYLE']._serialized_end=7666
- _globals['_SOURCE']._serialized_start=7669
- _globals['_SOURCE']._serialized_end=8454
- _globals['_SOURCE_PREPROCENTRY']._serialized_start=8313
- _globals['_SOURCE_PREPROCENTRY']._serialized_end=8397
- _globals['_SINK']._serialized_start=8457
- _globals['_SINK']._serialized_end=8951
- _globals['_SINK_RENAMESENTRY']._serialized_start=8869
- _globals['_SINK_RENAMESENTRY']._serialized_end=8915
+ DESCRIPTOR._loaded_options = None
+ _globals["_KAFKA"].fields_by_name["sasl_jaas_config"]._loaded_options = None
+ _globals["_KAFKA"].fields_by_name[
+ "sasl_jaas_config"
+ ]._serialized_options = b"\030\001"
+ _globals["_KAFKA"].fields_by_name["group_id"]._loaded_options = None
+ _globals["_KAFKA"].fields_by_name[
+ "group_id"
+ ]._serialized_options = b"\030\001"
+ _globals["_HTTPPATH_HEADERSENTRY"]._loaded_options = None
+ _globals["_HTTPPATH_HEADERSENTRY"]._serialized_options = b"8\001"
+ _globals["_SOURCE_PREPROCENTRY"]._loaded_options = None
+ _globals["_SOURCE_PREPROCENTRY"]._serialized_options = b"8\001"
+ _globals["_SINK_RENAMESENTRY"]._loaded_options = None
+ _globals["_SINK_RENAMESENTRY"]._serialized_options = b"8\001"
+ _globals["_CDCSTRATEGY"]._serialized_start = 9088
+ _globals["_CDCSTRATEGY"]._serialized_end = 9163
+ _globals["_EXTDATABASE"]._serialized_start = 207
+ _globals["_EXTDATABASE"]._serialized_end = 905
+ _globals["_SAMPLINGSTRATEGY"]._serialized_start = 907
+ _globals["_SAMPLINGSTRATEGY"]._serialized_end = 970
+ _globals["_PUBSUBFORMAT"]._serialized_start = 972
+ _globals["_PUBSUBFORMAT"]._serialized_end = 1049
+ _globals["_KAFKAFORMAT"]._serialized_start = 1052
+ _globals["_KAFKAFORMAT"]._serialized_end = 1240
+ _globals["_DEBEZIUMFORMAT"]._serialized_start = 1242
+ _globals["_DEBEZIUMFORMAT"]._serialized_end = 1258
+ _globals["_JSONFORMAT"]._serialized_start = 1260
+ _globals["_JSONFORMAT"]._serialized_end = 1272
+ _globals["_AVROFORMAT"]._serialized_start = 1274
+ _globals["_AVROFORMAT"]._serialized_end = 1357
+ _globals["_PROTOBUFFORMAT"]._serialized_start = 1359
+ _globals["_PROTOBUFFORMAT"]._serialized_end = 1446
+ _globals["_REFERENCE"]._serialized_start = 1449
+ _globals["_REFERENCE"]._serialized_end = 1671
+ _globals["_REFERENCE_EXTDBTYPE"]._serialized_start = 1524
+ _globals["_REFERENCE_EXTDBTYPE"]._serialized_end = 1671
+ _globals["_WEBHOOK"]._serialized_start = 1673
+ _globals["_WEBHOOK"]._serialized_end = 1742
+ _globals["_MYSQL"]._serialized_start = 1745
+ _globals["_MYSQL"]._serialized_end = 2013
+ _globals["_POSTGRES"]._serialized_start = 2016
+ _globals["_POSTGRES"]._serialized_end = 2287
+ _globals["_S3"]._serialized_start = 2290
+ _globals["_S3"]._serialized_end = 2594
+ _globals["_BIGQUERY"]._serialized_start = 2597
+ _globals["_BIGQUERY"]._serialized_end = 2779
+ _globals["_SNOWFLAKE"]._serialized_start = 2782
+ _globals["_SNOWFLAKE"]._serialized_end = 3071
+ _globals["_KAFKA"]._serialized_start = 3074
+ _globals["_KAFKA"]._serialized_end = 3451
+ _globals["_KINESIS"]._serialized_start = 3453
+ _globals["_KINESIS"]._serialized_end = 3480
+ _globals["_CREDENTIALS"]._serialized_start = 3483
+ _globals["_CREDENTIALS"]._serialized_end = 3694
+ _globals["_REDSHIFTAUTHENTICATION"]._serialized_start = 3696
+ _globals["_REDSHIFTAUTHENTICATION"]._serialized_end = 3821
+ _globals["_REDSHIFT"]._serialized_start = 3824
+ _globals["_REDSHIFT"]._serialized_end = 3977
+ _globals["_MONGO"]._serialized_start = 3980
+ _globals["_MONGO"]._serialized_end = 4213
+ _globals["_PUBSUB"]._serialized_start = 4216
+ _globals["_PUBSUB"]._serialized_end = 4376
+ _globals["_SENSITIVEDATUM"]._serialized_start = 4378
+ _globals["_SENSITIVEDATUM"]._serialized_end = 4493
+ _globals["_HTTP"]._serialized_start = 4496
+ _globals["_HTTP"]._serialized_end = 4680
+ _globals["_EXTTABLE"]._serialized_start = 4683
+ _globals["_EXTTABLE"]._serialized_end = 5442
+ _globals["_MYSQLTABLE"]._serialized_start = 5444
+ _globals["_MYSQLTABLE"]._serialized_end = 5525
+ _globals["_POSTGRESTABLE"]._serialized_start = 5527
+ _globals["_POSTGRESTABLE"]._serialized_end = 5649
+ _globals["_S3TABLE"]._serialized_start = 5652
+ _globals["_S3TABLE"]._serialized_end = 5899
+ _globals["_KAFKATOPIC"]._serialized_start = 5902
+ _globals["_KAFKATOPIC"]._serialized_end = 6031
+ _globals["_BIGQUERYTABLE"]._serialized_start = 6033
+ _globals["_BIGQUERYTABLE"]._serialized_end = 6117
+ _globals["_SNOWFLAKETABLE"]._serialized_start = 6119
+ _globals["_SNOWFLAKETABLE"]._serialized_end = 6204
+ _globals["_WEBHOOKENDPOINT"]._serialized_start = 6207
+ _globals["_WEBHOOKENDPOINT"]._serialized_end = 6336
+ _globals["_KINESISSTREAM"]._serialized_start = 6339
+ _globals["_KINESISSTREAM"]._serialized_end = 6550
+ _globals["_REDSHIFTTABLE"]._serialized_start = 6552
+ _globals["_REDSHIFTTABLE"]._serialized_end = 6636
+ _globals["_PUBSUBTOPIC"]._serialized_start = 6639
+ _globals["_PUBSUBTOPIC"]._serialized_end = 6773
+ _globals["_HTTPPATH"]._serialized_start = 6776
+ _globals["_HTTPPATH"]._serialized_end = 6995
+ _globals["_HTTPPATH_HEADERSENTRY"]._serialized_start = 6939
+ _globals["_HTTPPATH_HEADERSENTRY"]._serialized_end = 6985
+ _globals["_EVAL"]._serialized_start = 6998
+ _globals["_EVAL"]._serialized_end = 7156
+ _globals["_PREPROCVALUE"]._serialized_start = 7159
+ _globals["_PREPROCVALUE"]._serialized_end = 7290
+ _globals["_MONGOCOLLECTION"]._serialized_start = 7292
+ _globals["_MONGOCOLLECTION"]._serialized_end = 7383
+ _globals["_SNAPSHOTDATA"]._serialized_start = 7385
+ _globals["_SNAPSHOTDATA"]._serialized_end = 7435
+ _globals["_INCREMENTAL"]._serialized_start = 7437
+ _globals["_INCREMENTAL"]._serialized_end = 7450
+ _globals["_RECREATE"]._serialized_start = 7452
+ _globals["_RECREATE"]._serialized_end = 7462
+ _globals["_STYLE"]._serialized_start = 7465
+ _globals["_STYLE"]._serialized_end = 7653
+ _globals["_SOURCE"]._serialized_start = 7656
+ _globals["_SOURCE"]._serialized_end = 8589
+ _globals["_SOURCE_PREPROCENTRY"]._serialized_start = 8414
+ _globals["_SOURCE_PREPROCENTRY"]._serialized_end = 8498
+ _globals["_SINK"]._serialized_start = 8592
+ _globals["_SINK"]._serialized_end = 9086
+ _globals["_SINK_RENAMESENTRY"]._serialized_start = 9004
+ _globals["_SINK_RENAMESENTRY"]._serialized_end = 9050
# @@protoc_insertion_point(module_scope)
diff --git a/fennel/gen/connector_pb2.pyi b/fennel/gen/connector_pb2.pyi
index 12ea251ef..551e7ed68 100644
--- a/fennel/gen/connector_pb2.pyi
+++ b/fennel/gen/connector_pb2.pyi
@@ -11,6 +11,7 @@ database
4. Sink refers to a (table, dataset) along with some config
5. This whole module is called connector.
"""
+
import builtins
import collections.abc
import expression_pb2
@@ -39,7 +40,12 @@ class _CDCStrategy:
ValueType = typing.NewType("ValueType", builtins.int)
V: typing_extensions.TypeAlias = ValueType
-class _CDCStrategyEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_CDCStrategy.ValueType], builtins.type):
+class _CDCStrategyEnumTypeWrapper(
+ google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[
+ _CDCStrategy.ValueType
+ ],
+ builtins.type,
+):
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
Append: _CDCStrategy.ValueType # 0
Upsert: _CDCStrategy.ValueType # 1
@@ -89,6 +95,7 @@ class ExtDatabase(google.protobuf.message.Message):
"""When a source has already been created on the console
Or via code and is specified by name ONLY.
"""
+
@property
def s3(self) -> global___S3: ...
@property
@@ -127,9 +134,94 @@ class ExtDatabase(google.protobuf.message.Message):
pubsub: global___PubSub | None = ...,
http: global___Http | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["bigquery", b"bigquery", "http", b"http", "kafka", b"kafka", "kinesis", b"kinesis", "mongo", b"mongo", "mysql", b"mysql", "postgres", b"postgres", "pubsub", b"pubsub", "redshift", b"redshift", "reference", b"reference", "s3", b"s3", "snowflake", b"snowflake", "variant", b"variant", "webhook", b"webhook"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["bigquery", b"bigquery", "http", b"http", "kafka", b"kafka", "kinesis", b"kinesis", "mongo", b"mongo", "mysql", b"mysql", "name", b"name", "postgres", b"postgres", "pubsub", b"pubsub", "redshift", b"redshift", "reference", b"reference", "s3", b"s3", "snowflake", b"snowflake", "variant", b"variant", "webhook", b"webhook"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["variant", b"variant"]) -> typing_extensions.Literal["mysql", "postgres", "reference", "s3", "bigquery", "snowflake", "kafka", "webhook", "kinesis", "redshift", "mongo", "pubsub", "http"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "bigquery",
+ b"bigquery",
+ "http",
+ b"http",
+ "kafka",
+ b"kafka",
+ "kinesis",
+ b"kinesis",
+ "mongo",
+ b"mongo",
+ "mysql",
+ b"mysql",
+ "postgres",
+ b"postgres",
+ "pubsub",
+ b"pubsub",
+ "redshift",
+ b"redshift",
+ "reference",
+ b"reference",
+ "s3",
+ b"s3",
+ "snowflake",
+ b"snowflake",
+ "variant",
+ b"variant",
+ "webhook",
+ b"webhook",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "bigquery",
+ b"bigquery",
+ "http",
+ b"http",
+ "kafka",
+ b"kafka",
+ "kinesis",
+ b"kinesis",
+ "mongo",
+ b"mongo",
+ "mysql",
+ b"mysql",
+ "name",
+ b"name",
+ "postgres",
+ b"postgres",
+ "pubsub",
+ b"pubsub",
+ "redshift",
+ b"redshift",
+ "reference",
+ b"reference",
+ "s3",
+ b"s3",
+ "snowflake",
+ b"snowflake",
+ "variant",
+ b"variant",
+ "webhook",
+ b"webhook",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["variant", b"variant"]
+ ) -> (
+ typing_extensions.Literal[
+ "mysql",
+ "postgres",
+ "reference",
+ "s3",
+ "bigquery",
+ "snowflake",
+ "kafka",
+ "webhook",
+ "kinesis",
+ "redshift",
+ "mongo",
+ "pubsub",
+ "http",
+ ]
+ | None
+ ): ...
global___ExtDatabase = ExtDatabase
@@ -141,15 +233,25 @@ class SamplingStrategy(google.protobuf.message.Message):
COLUMNS_USED_FIELD_NUMBER: builtins.int
sampling_rate: builtins.float
@property
- def columns_used(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
+ def columns_used(
+ self,
+ ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[
+ builtins.str
+ ]:
"""columns_used is a list of columns used for sampling"""
+
def __init__(
self,
*,
sampling_rate: builtins.float = ...,
columns_used: collections.abc.Iterable[builtins.str] | None = ...,
) -> None: ...
- def ClearField(self, field_name: typing_extensions.Literal["columns_used", b"columns_used", "sampling_rate", b"sampling_rate"]) -> None: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "columns_used", b"columns_used", "sampling_rate", b"sampling_rate"
+ ],
+ ) -> None: ...
global___SamplingStrategy = SamplingStrategy
@@ -165,9 +267,21 @@ class PubSubFormat(google.protobuf.message.Message):
*,
json: global___JsonFormat | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["json", b"json", "variant", b"variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["json", b"json", "variant", b"variant"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["variant", b"variant"]) -> typing_extensions.Literal["json"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "json", b"json", "variant", b"variant"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "json", b"json", "variant", b"variant"
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["variant", b"variant"]
+ ) -> typing_extensions.Literal["json"] | None: ...
global___PubSubFormat = PubSubFormat
@@ -191,9 +305,35 @@ class KafkaFormat(google.protobuf.message.Message):
avro: global___AvroFormat | None = ...,
protobuf: global___ProtobufFormat | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["avro", b"avro", "json", b"json", "protobuf", b"protobuf", "variant", b"variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["avro", b"avro", "json", b"json", "protobuf", b"protobuf", "variant", b"variant"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["variant", b"variant"]) -> typing_extensions.Literal["json", "avro", "protobuf"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "avro",
+ b"avro",
+ "json",
+ b"json",
+ "protobuf",
+ b"protobuf",
+ "variant",
+ b"variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "avro",
+ b"avro",
+ "json",
+ b"json",
+ "protobuf",
+ b"protobuf",
+ "variant",
+ b"variant",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["variant", b"variant"]
+ ) -> typing_extensions.Literal["json", "avro", "protobuf"] | None: ...
global___KafkaFormat = KafkaFormat
@@ -229,8 +369,18 @@ class AvroFormat(google.protobuf.message.Message):
*,
schema_registry: schema_registry_pb2.SchemaRegistry | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["schema_registry", b"schema_registry"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["schema_registry", b"schema_registry"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "schema_registry", b"schema_registry"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "schema_registry", b"schema_registry"
+ ],
+ ) -> None: ...
global___AvroFormat = AvroFormat
@@ -246,8 +396,18 @@ class ProtobufFormat(google.protobuf.message.Message):
*,
schema_registry: schema_registry_pb2.SchemaRegistry | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["schema_registry", b"schema_registry"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["schema_registry", b"schema_registry"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "schema_registry", b"schema_registry"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "schema_registry", b"schema_registry"
+ ],
+ ) -> None: ...
global___ProtobufFormat = ProtobufFormat
@@ -259,7 +419,12 @@ class Reference(google.protobuf.message.Message):
ValueType = typing.NewType("ValueType", builtins.int)
V: typing_extensions.TypeAlias = ValueType
- class _ExtDBTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[Reference._ExtDBType.ValueType], builtins.type): # noqa: F821
+ class _ExtDBTypeEnumTypeWrapper(
+ google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[
+ Reference._ExtDBType.ValueType
+ ],
+ builtins.type,
+ ): # noqa: F821
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
MYSQL: Reference._ExtDBType.ValueType # 0
POSTGRES: Reference._ExtDBType.ValueType # 1
@@ -293,7 +458,9 @@ class Reference(google.protobuf.message.Message):
*,
dbtype: global___Reference.ExtDBType.ValueType = ...,
) -> None: ...
- def ClearField(self, field_name: typing_extensions.Literal["dbtype", b"dbtype"]) -> None: ...
+ def ClearField(
+ self, field_name: typing_extensions.Literal["dbtype", b"dbtype"]
+ ) -> None: ...
global___Reference = Reference
@@ -312,8 +479,15 @@ class Webhook(google.protobuf.message.Message):
name: builtins.str = ...,
retention: google.protobuf.duration_pb2.Duration | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["retention", b"retention"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["name", b"name", "retention", b"retention"]) -> None: ...
+ def HasField(
+ self, field_name: typing_extensions.Literal["retention", b"retention"]
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "name", b"name", "retention", b"retention"
+ ],
+ ) -> None: ...
global___Webhook = Webhook
@@ -352,12 +526,62 @@ class MySQL(google.protobuf.message.Message):
port: builtins.int = ...,
jdbc_params: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "jdbc_params", b"jdbc_params", "password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "port", b"port", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "user",
+ b"user",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "database",
+ b"database",
+ "host",
+ b"host",
+ "jdbc_params",
+ b"jdbc_params",
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "port",
+ b"port",
+ "user",
+ b"user",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "password_variant", b"password_variant"
+ ],
+ ) -> typing_extensions.Literal["password", "password_secret"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["user", "username_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "username_variant", b"username_variant"
+ ],
+ ) -> typing_extensions.Literal["user", "username_secret"] | None: ...
global___MySQL = MySQL
@@ -396,12 +620,62 @@ class Postgres(google.protobuf.message.Message):
port: builtins.int = ...,
jdbc_params: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "jdbc_params", b"jdbc_params", "password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "port", b"port", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "user",
+ b"user",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "database",
+ b"database",
+ "host",
+ b"host",
+ "jdbc_params",
+ b"jdbc_params",
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "port",
+ b"port",
+ "user",
+ b"user",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "password_variant", b"password_variant"
+ ],
+ ) -> typing_extensions.Literal["password", "password_secret"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["user", "username_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "username_variant", b"username_variant"
+ ],
+ ) -> typing_extensions.Literal["user", "username_secret"] | None: ...
global___Postgres = Postgres
@@ -431,14 +705,76 @@ class S3(google.protobuf.message.Message):
aws_access_key_id_secret: secret_pb2.SecretRef | None = ...,
role_arn: builtins.str | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["_role_arn", b"_role_arn", "aws_access_key_id", b"aws_access_key_id", "aws_access_key_id_secret", b"aws_access_key_id_secret", "aws_access_key_id_variant", b"aws_access_key_id_variant", "aws_secret_access_key", b"aws_secret_access_key", "aws_secret_access_key_secret", b"aws_secret_access_key_secret", "aws_secret_access_key_variant", b"aws_secret_access_key_variant", "role_arn", b"role_arn"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["_role_arn", b"_role_arn", "aws_access_key_id", b"aws_access_key_id", "aws_access_key_id_secret", b"aws_access_key_id_secret", "aws_access_key_id_variant", b"aws_access_key_id_variant", "aws_secret_access_key", b"aws_secret_access_key", "aws_secret_access_key_secret", b"aws_secret_access_key_secret", "aws_secret_access_key_variant", b"aws_secret_access_key_variant", "role_arn", b"role_arn"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_role_arn",
+ b"_role_arn",
+ "aws_access_key_id",
+ b"aws_access_key_id",
+ "aws_access_key_id_secret",
+ b"aws_access_key_id_secret",
+ "aws_access_key_id_variant",
+ b"aws_access_key_id_variant",
+ "aws_secret_access_key",
+ b"aws_secret_access_key",
+ "aws_secret_access_key_secret",
+ b"aws_secret_access_key_secret",
+ "aws_secret_access_key_variant",
+ b"aws_secret_access_key_variant",
+ "role_arn",
+ b"role_arn",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_role_arn",
+ b"_role_arn",
+ "aws_access_key_id",
+ b"aws_access_key_id",
+ "aws_access_key_id_secret",
+ b"aws_access_key_id_secret",
+ "aws_access_key_id_variant",
+ b"aws_access_key_id_variant",
+ "aws_secret_access_key",
+ b"aws_secret_access_key",
+ "aws_secret_access_key_secret",
+ b"aws_secret_access_key_secret",
+ "aws_secret_access_key_variant",
+ b"aws_secret_access_key_variant",
+ "role_arn",
+ b"role_arn",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_role_arn", b"_role_arn"]) -> typing_extensions.Literal["role_arn"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_role_arn", b"_role_arn"]
+ ) -> typing_extensions.Literal["role_arn"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["aws_access_key_id_variant", b"aws_access_key_id_variant"]) -> typing_extensions.Literal["aws_access_key_id", "aws_access_key_id_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "aws_access_key_id_variant", b"aws_access_key_id_variant"
+ ],
+ ) -> (
+ typing_extensions.Literal[
+ "aws_access_key_id", "aws_access_key_id_secret"
+ ]
+ | None
+ ): ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["aws_secret_access_key_variant", b"aws_secret_access_key_variant"]) -> typing_extensions.Literal["aws_secret_access_key", "aws_secret_access_key_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "aws_secret_access_key_variant", b"aws_secret_access_key_variant"
+ ],
+ ) -> (
+ typing_extensions.Literal[
+ "aws_secret_access_key", "aws_secret_access_key_secret"
+ ]
+ | None
+ ): ...
global___S3 = S3
@@ -464,9 +800,43 @@ class Bigquery(google.protobuf.message.Message):
dataset_id: builtins.str = ...,
project_id: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["service_account_key", b"service_account_key", "service_account_key_secret", b"service_account_key_secret", "service_account_key_variant", b"service_account_key_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["dataset_id", b"dataset_id", "project_id", b"project_id", "service_account_key", b"service_account_key", "service_account_key_secret", b"service_account_key_secret", "service_account_key_variant", b"service_account_key_variant"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["service_account_key_variant", b"service_account_key_variant"]) -> typing_extensions.Literal["service_account_key", "service_account_key_secret"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "service_account_key",
+ b"service_account_key",
+ "service_account_key_secret",
+ b"service_account_key_secret",
+ "service_account_key_variant",
+ b"service_account_key_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "dataset_id",
+ b"dataset_id",
+ "project_id",
+ b"project_id",
+ "service_account_key",
+ b"service_account_key",
+ "service_account_key_secret",
+ b"service_account_key_secret",
+ "service_account_key_variant",
+ b"service_account_key_variant",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "service_account_key_variant", b"service_account_key_variant"
+ ],
+ ) -> (
+ typing_extensions.Literal[
+ "service_account_key", "service_account_key_secret"
+ ]
+ | None
+ ): ...
global___Bigquery = Bigquery
@@ -508,12 +878,64 @@ class Snowflake(google.protobuf.message.Message):
role: builtins.str = ...,
database: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["account", b"account", "database", b"database", "password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "role", b"role", "schema", b"schema", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant", "warehouse", b"warehouse"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "user",
+ b"user",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "account",
+ b"account",
+ "database",
+ b"database",
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "role",
+ b"role",
+ "schema",
+ b"schema",
+ "user",
+ b"user",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ "warehouse",
+ b"warehouse",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "password_variant", b"password_variant"
+ ],
+ ) -> typing_extensions.Literal["password", "password_secret"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["user", "username_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "username_variant", b"username_variant"
+ ],
+ ) -> typing_extensions.Literal["user", "username_secret"] | None: ...
global___Snowflake = Snowflake
@@ -555,12 +977,70 @@ class Kafka(google.protobuf.message.Message):
sasl_jaas_config: builtins.str = ...,
group_id: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["sasl_password_secret", b"sasl_password_secret", "sasl_password_variant", b"sasl_password_variant", "sasl_plain_password", b"sasl_plain_password", "sasl_plain_username", b"sasl_plain_username", "sasl_username_secret", b"sasl_username_secret", "sasl_username_variant", b"sasl_username_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["bootstrap_servers", b"bootstrap_servers", "group_id", b"group_id", "sasl_jaas_config", b"sasl_jaas_config", "sasl_mechanism", b"sasl_mechanism", "sasl_password_secret", b"sasl_password_secret", "sasl_password_variant", b"sasl_password_variant", "sasl_plain_password", b"sasl_plain_password", "sasl_plain_username", b"sasl_plain_username", "sasl_username_secret", b"sasl_username_secret", "sasl_username_variant", b"sasl_username_variant", "security_protocol", b"security_protocol"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "sasl_password_secret",
+ b"sasl_password_secret",
+ "sasl_password_variant",
+ b"sasl_password_variant",
+ "sasl_plain_password",
+ b"sasl_plain_password",
+ "sasl_plain_username",
+ b"sasl_plain_username",
+ "sasl_username_secret",
+ b"sasl_username_secret",
+ "sasl_username_variant",
+ b"sasl_username_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "bootstrap_servers",
+ b"bootstrap_servers",
+ "group_id",
+ b"group_id",
+ "sasl_jaas_config",
+ b"sasl_jaas_config",
+ "sasl_mechanism",
+ b"sasl_mechanism",
+ "sasl_password_secret",
+ b"sasl_password_secret",
+ "sasl_password_variant",
+ b"sasl_password_variant",
+ "sasl_plain_password",
+ b"sasl_plain_password",
+ "sasl_plain_username",
+ b"sasl_plain_username",
+ "sasl_username_secret",
+ b"sasl_username_secret",
+ "sasl_username_variant",
+ b"sasl_username_variant",
+ "security_protocol",
+ b"security_protocol",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["sasl_password_variant", b"sasl_password_variant"]) -> typing_extensions.Literal["sasl_plain_password", "sasl_password_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "sasl_password_variant", b"sasl_password_variant"
+ ],
+ ) -> (
+ typing_extensions.Literal["sasl_plain_password", "sasl_password_secret"]
+ | None
+ ): ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["sasl_username_variant", b"sasl_username_variant"]) -> typing_extensions.Literal["sasl_plain_username", "sasl_username_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "sasl_username_variant", b"sasl_username_variant"
+ ],
+ ) -> (
+ typing_extensions.Literal["sasl_plain_username", "sasl_username_secret"]
+ | None
+ ): ...
global___Kafka = Kafka
@@ -575,7 +1055,9 @@ class Kinesis(google.protobuf.message.Message):
*,
role_arn: builtins.str = ...,
) -> None: ...
- def ClearField(self, field_name: typing_extensions.Literal["role_arn", b"role_arn"]) -> None: ...
+ def ClearField(
+ self, field_name: typing_extensions.Literal["role_arn", b"role_arn"]
+ ) -> None: ...
global___Kinesis = Kinesis
@@ -603,12 +1085,54 @@ class Credentials(google.protobuf.message.Message):
password: builtins.str = ...,
password_secret: secret_pb2.SecretRef | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "username", b"username", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "username", b"username", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "username",
+ b"username",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "username",
+ b"username",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "password_variant", b"password_variant"
+ ],
+ ) -> typing_extensions.Literal["password", "password_secret"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["username", "username_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "username_variant", b"username_variant"
+ ],
+ ) -> typing_extensions.Literal["username", "username_secret"] | None: ...
global___Credentials = Credentials
@@ -627,9 +1151,33 @@ class RedshiftAuthentication(google.protobuf.message.Message):
s3_access_role_arn: builtins.str = ...,
credentials: global___Credentials | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["credentials", b"credentials", "s3_access_role_arn", b"s3_access_role_arn", "variant", b"variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["credentials", b"credentials", "s3_access_role_arn", b"s3_access_role_arn", "variant", b"variant"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["variant", b"variant"]) -> typing_extensions.Literal["s3_access_role_arn", "credentials"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "credentials",
+ b"credentials",
+ "s3_access_role_arn",
+ b"s3_access_role_arn",
+ "variant",
+ b"variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "credentials",
+ b"credentials",
+ "s3_access_role_arn",
+ b"s3_access_role_arn",
+ "variant",
+ b"variant",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["variant", b"variant"]
+ ) -> (
+ typing_extensions.Literal["s3_access_role_arn", "credentials"] | None
+ ): ...
global___RedshiftAuthentication = RedshiftAuthentication
@@ -657,8 +1205,27 @@ class Redshift(google.protobuf.message.Message):
schema: builtins.str = ...,
redshift_authentication: global___RedshiftAuthentication | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["redshift_authentication", b"redshift_authentication"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "port", b"port", "redshift_authentication", b"redshift_authentication", "schema", b"schema"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "redshift_authentication", b"redshift_authentication"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "database",
+ b"database",
+ "host",
+ b"host",
+ "port",
+ b"port",
+ "redshift_authentication",
+ b"redshift_authentication",
+ "schema",
+ b"schema",
+ ],
+ ) -> None: ...
global___Redshift = Redshift
@@ -691,12 +1258,58 @@ class Mongo(google.protobuf.message.Message):
host: builtins.str = ...,
database: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["database", b"database", "host", b"host", "password", b"password", "password_secret", b"password_secret", "password_variant", b"password_variant", "user", b"user", "username_secret", b"username_secret", "username_variant", b"username_variant"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "user",
+ b"user",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "database",
+ b"database",
+ "host",
+ b"host",
+ "password",
+ b"password",
+ "password_secret",
+ b"password_secret",
+ "password_variant",
+ b"password_variant",
+ "user",
+ b"user",
+ "username_secret",
+ b"username_secret",
+ "username_variant",
+ b"username_variant",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["password_variant", b"password_variant"]) -> typing_extensions.Literal["password", "password_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "password_variant", b"password_variant"
+ ],
+ ) -> typing_extensions.Literal["password", "password_secret"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["username_variant", b"username_variant"]) -> typing_extensions.Literal["user", "username_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "username_variant", b"username_variant"
+ ],
+ ) -> typing_extensions.Literal["user", "username_secret"] | None: ...
global___Mongo = Mongo
@@ -719,9 +1332,41 @@ class PubSub(google.protobuf.message.Message):
service_account_key_secret: secret_pb2.SecretRef | None = ...,
project_id: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["service_account_key", b"service_account_key", "service_account_key_secret", b"service_account_key_secret", "service_account_key_variant", b"service_account_key_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["project_id", b"project_id", "service_account_key", b"service_account_key", "service_account_key_secret", b"service_account_key_secret", "service_account_key_variant", b"service_account_key_variant"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["service_account_key_variant", b"service_account_key_variant"]) -> typing_extensions.Literal["service_account_key", "service_account_key_secret"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "service_account_key",
+ b"service_account_key",
+ "service_account_key_secret",
+ b"service_account_key_secret",
+ "service_account_key_variant",
+ b"service_account_key_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "project_id",
+ b"project_id",
+ "service_account_key",
+ b"service_account_key",
+ "service_account_key_secret",
+ b"service_account_key_secret",
+ "service_account_key_variant",
+ b"service_account_key_variant",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "service_account_key_variant", b"service_account_key_variant"
+ ],
+ ) -> (
+ typing_extensions.Literal[
+ "service_account_key", "service_account_key_secret"
+ ]
+ | None
+ ): ...
global___PubSub = PubSub
@@ -740,9 +1385,34 @@ class SensitiveDatum(google.protobuf.message.Message):
secret: builtins.str = ...,
secret_ref: secret_pb2.SecretRef | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["secret", b"secret", "secret_ref", b"secret_ref", "sensitive_datum_variant", b"sensitive_datum_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["secret", b"secret", "secret_ref", b"secret_ref", "sensitive_datum_variant", b"sensitive_datum_variant"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["sensitive_datum_variant", b"sensitive_datum_variant"]) -> typing_extensions.Literal["secret", "secret_ref"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "secret",
+ b"secret",
+ "secret_ref",
+ b"secret_ref",
+ "sensitive_datum_variant",
+ b"sensitive_datum_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "secret",
+ b"secret",
+ "secret_ref",
+ b"secret_ref",
+ "sensitive_datum_variant",
+ b"sensitive_datum_variant",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "sensitive_datum_variant", b"sensitive_datum_variant"
+ ],
+ ) -> typing_extensions.Literal["secret", "secret_ref"] | None: ...
global___SensitiveDatum = SensitiveDatum
@@ -763,6 +1433,7 @@ class Http(google.protobuf.message.Message):
"""Making this optional for now since local testing doesn't require it
Next id: 5
"""
+
def __init__(
self,
*,
@@ -771,12 +1442,47 @@ class Http(google.protobuf.message.Message):
healthz: builtins.str = ...,
ca_cert: global___SensitiveDatum | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["_ca_cert", b"_ca_cert", "ca_cert", b"ca_cert", "host", b"host", "host_secret", b"host_secret", "host_variant", b"host_variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["_ca_cert", b"_ca_cert", "ca_cert", b"ca_cert", "healthz", b"healthz", "host", b"host", "host_secret", b"host_secret", "host_variant", b"host_variant"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_ca_cert",
+ b"_ca_cert",
+ "ca_cert",
+ b"ca_cert",
+ "host",
+ b"host",
+ "host_secret",
+ b"host_secret",
+ "host_variant",
+ b"host_variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_ca_cert",
+ b"_ca_cert",
+ "ca_cert",
+ b"ca_cert",
+ "healthz",
+ b"healthz",
+ "host",
+ b"host",
+ "host_secret",
+ b"host_secret",
+ "host_variant",
+ b"host_variant",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_ca_cert", b"_ca_cert"]) -> typing_extensions.Literal["ca_cert"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_ca_cert", b"_ca_cert"]
+ ) -> typing_extensions.Literal["ca_cert"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["host_variant", b"host_variant"]) -> typing_extensions.Literal["host", "host_secret"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal["host_variant", b"host_variant"],
+ ) -> typing_extensions.Literal["host", "host_secret"] | None: ...
global___Http = Http
@@ -841,9 +1547,87 @@ class ExtTable(google.protobuf.message.Message):
pubsub_topic: global___PubSubTopic | None = ...,
http_path: global___HttpPath | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["bigquery_table", b"bigquery_table", "endpoint", b"endpoint", "http_path", b"http_path", "kafka_topic", b"kafka_topic", "kinesis_stream", b"kinesis_stream", "mongo_collection", b"mongo_collection", "mysql_table", b"mysql_table", "pg_table", b"pg_table", "pubsub_topic", b"pubsub_topic", "redshift_table", b"redshift_table", "s3_table", b"s3_table", "snowflake_table", b"snowflake_table", "variant", b"variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["bigquery_table", b"bigquery_table", "endpoint", b"endpoint", "http_path", b"http_path", "kafka_topic", b"kafka_topic", "kinesis_stream", b"kinesis_stream", "mongo_collection", b"mongo_collection", "mysql_table", b"mysql_table", "pg_table", b"pg_table", "pubsub_topic", b"pubsub_topic", "redshift_table", b"redshift_table", "s3_table", b"s3_table", "snowflake_table", b"snowflake_table", "variant", b"variant"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["variant", b"variant"]) -> typing_extensions.Literal["mysql_table", "pg_table", "s3_table", "kafka_topic", "snowflake_table", "bigquery_table", "endpoint", "kinesis_stream", "redshift_table", "mongo_collection", "pubsub_topic", "http_path"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "bigquery_table",
+ b"bigquery_table",
+ "endpoint",
+ b"endpoint",
+ "http_path",
+ b"http_path",
+ "kafka_topic",
+ b"kafka_topic",
+ "kinesis_stream",
+ b"kinesis_stream",
+ "mongo_collection",
+ b"mongo_collection",
+ "mysql_table",
+ b"mysql_table",
+ "pg_table",
+ b"pg_table",
+ "pubsub_topic",
+ b"pubsub_topic",
+ "redshift_table",
+ b"redshift_table",
+ "s3_table",
+ b"s3_table",
+ "snowflake_table",
+ b"snowflake_table",
+ "variant",
+ b"variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "bigquery_table",
+ b"bigquery_table",
+ "endpoint",
+ b"endpoint",
+ "http_path",
+ b"http_path",
+ "kafka_topic",
+ b"kafka_topic",
+ "kinesis_stream",
+ b"kinesis_stream",
+ "mongo_collection",
+ b"mongo_collection",
+ "mysql_table",
+ b"mysql_table",
+ "pg_table",
+ b"pg_table",
+ "pubsub_topic",
+ b"pubsub_topic",
+ "redshift_table",
+ b"redshift_table",
+ "s3_table",
+ b"s3_table",
+ "snowflake_table",
+ b"snowflake_table",
+ "variant",
+ b"variant",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["variant", b"variant"]
+ ) -> (
+ typing_extensions.Literal[
+ "mysql_table",
+ "pg_table",
+ "s3_table",
+ "kafka_topic",
+ "snowflake_table",
+ "bigquery_table",
+ "endpoint",
+ "kinesis_stream",
+ "redshift_table",
+ "mongo_collection",
+ "pubsub_topic",
+ "http_path",
+ ]
+ | None
+ ): ...
global___ExtTable = ExtTable
@@ -862,8 +1646,15 @@ class MySQLTable(google.protobuf.message.Message):
db: global___ExtDatabase | None = ...,
table_name: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["db", b"db", "table_name", b"table_name"]) -> None: ...
+ def HasField(
+ self, field_name: typing_extensions.Literal["db", b"db"]
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "table_name", b"table_name"
+ ],
+ ) -> None: ...
global___MySQLTable = MySQLTable
@@ -885,9 +1676,29 @@ class PostgresTable(google.protobuf.message.Message):
table_name: builtins.str = ...,
slot_name: builtins.str | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["_slot_name", b"_slot_name", "db", b"db", "slot_name", b"slot_name"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["_slot_name", b"_slot_name", "db", b"db", "slot_name", b"slot_name", "table_name", b"table_name"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_slot_name", b"_slot_name"]) -> typing_extensions.Literal["slot_name"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_slot_name", b"_slot_name", "db", b"db", "slot_name", b"slot_name"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_slot_name",
+ b"_slot_name",
+ "db",
+ b"db",
+ "slot_name",
+ b"slot_name",
+ "table_name",
+ b"table_name",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal["_slot_name", b"_slot_name"],
+ ) -> typing_extensions.Literal["slot_name"] | None: ...
global___PostgresTable = PostgresTable
@@ -915,7 +1726,11 @@ class S3Table(google.protobuf.message.Message):
@property
def spread(self) -> google.protobuf.duration_pb2.Duration: ...
@property
- def headers(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+ def headers(
+ self,
+ ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[
+ builtins.str
+ ]: ...
def __init__(
self,
*,
@@ -929,9 +1744,40 @@ class S3Table(google.protobuf.message.Message):
spread: google.protobuf.duration_pb2.Duration | None = ...,
headers: collections.abc.Iterable[builtins.str] | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["_spread", b"_spread", "db", b"db", "spread", b"spread"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["_spread", b"_spread", "bucket", b"bucket", "db", b"db", "delimiter", b"delimiter", "format", b"format", "headers", b"headers", "path_prefix", b"path_prefix", "path_suffix", b"path_suffix", "pre_sorted", b"pre_sorted", "spread", b"spread"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_spread", b"_spread"]) -> typing_extensions.Literal["spread"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_spread", b"_spread", "db", b"db", "spread", b"spread"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_spread",
+ b"_spread",
+ "bucket",
+ b"bucket",
+ "db",
+ b"db",
+ "delimiter",
+ b"delimiter",
+ "format",
+ b"format",
+ "headers",
+ b"headers",
+ "path_prefix",
+ b"path_prefix",
+ "path_suffix",
+ b"path_suffix",
+ "pre_sorted",
+ b"pre_sorted",
+ "spread",
+ b"spread",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_spread", b"_spread"]
+ ) -> typing_extensions.Literal["spread"] | None: ...
global___S3Table = S3Table
@@ -954,8 +1800,16 @@ class KafkaTopic(google.protobuf.message.Message):
topic: builtins.str = ...,
format: global___KafkaFormat | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db", "format", b"format"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["db", b"db", "format", b"format", "topic", b"topic"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal["db", b"db", "format", b"format"],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "format", b"format", "topic", b"topic"
+ ],
+ ) -> None: ...
global___KafkaTopic = KafkaTopic
@@ -974,8 +1828,15 @@ class BigqueryTable(google.protobuf.message.Message):
db: global___ExtDatabase | None = ...,
table_name: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["db", b"db", "table_name", b"table_name"]) -> None: ...
+ def HasField(
+ self, field_name: typing_extensions.Literal["db", b"db"]
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "table_name", b"table_name"
+ ],
+ ) -> None: ...
global___BigqueryTable = BigqueryTable
@@ -994,8 +1855,15 @@ class SnowflakeTable(google.protobuf.message.Message):
db: global___ExtDatabase | None = ...,
table_name: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["db", b"db", "table_name", b"table_name"]) -> None: ...
+ def HasField(
+ self, field_name: typing_extensions.Literal["db", b"db"]
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "table_name", b"table_name"
+ ],
+ ) -> None: ...
global___SnowflakeTable = SnowflakeTable
@@ -1018,8 +1886,18 @@ class WebhookEndpoint(google.protobuf.message.Message):
endpoint: builtins.str = ...,
duration: google.protobuf.duration_pb2.Duration | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db", "duration", b"duration"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["db", b"db", "duration", b"duration", "endpoint", b"endpoint"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "duration", b"duration"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "duration", b"duration", "endpoint", b"endpoint"
+ ],
+ ) -> None: ...
global___WebhookEndpoint = WebhookEndpoint
@@ -1048,8 +1926,27 @@ class KinesisStream(google.protobuf.message.Message):
format: builtins.str = ...,
db: global___ExtDatabase | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db", "init_timestamp", b"init_timestamp"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["db", b"db", "format", b"format", "init_position", b"init_position", "init_timestamp", b"init_timestamp", "stream_arn", b"stream_arn"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "init_timestamp", b"init_timestamp"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db",
+ b"db",
+ "format",
+ b"format",
+ "init_position",
+ b"init_position",
+ "init_timestamp",
+ b"init_timestamp",
+ "stream_arn",
+ b"stream_arn",
+ ],
+ ) -> None: ...
global___KinesisStream = KinesisStream
@@ -1068,8 +1965,15 @@ class RedshiftTable(google.protobuf.message.Message):
db: global___ExtDatabase | None = ...,
table_name: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["db", b"db", "table_name", b"table_name"]) -> None: ...
+ def HasField(
+ self, field_name: typing_extensions.Literal["db", b"db"]
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "table_name", b"table_name"
+ ],
+ ) -> None: ...
global___RedshiftTable = RedshiftTable
@@ -1092,8 +1996,16 @@ class PubSubTopic(google.protobuf.message.Message):
topic_id: builtins.str = ...,
format: global___PubSubFormat | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db", "format", b"format"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["db", b"db", "format", b"format", "topic_id", b"topic_id"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal["db", b"db", "format", b"format"],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "db", b"db", "format", b"format", "topic_id", b"topic_id"
+ ],
+ ) -> None: ...
global___PubSubTopic = PubSubTopic
@@ -1115,7 +2027,12 @@ class HttpPath(google.protobuf.message.Message):
key: builtins.str = ...,
value: builtins.str = ...,
) -> None: ...
- def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "key", b"key", "value", b"value"
+ ],
+ ) -> None: ...
DB_FIELD_NUMBER: builtins.int
ENDPOINT_FIELD_NUMBER: builtins.int
@@ -1126,48 +2043,103 @@ class HttpPath(google.protobuf.message.Message):
endpoint: builtins.str
limit: builtins.int
@property
- def headers(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ...
+ def headers(
+ self,
+ ) -> google.protobuf.internal.containers.ScalarMap[
+ builtins.str, builtins.str
+ ]: ...
def __init__(
self,
*,
db: global___ExtDatabase | None = ...,
endpoint: builtins.str = ...,
limit: builtins.int | None = ...,
- headers: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
+ headers: (
+ collections.abc.Mapping[builtins.str, builtins.str] | None
+ ) = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["_limit", b"_limit", "db", b"db", "limit", b"limit"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["_limit", b"_limit", "db", b"db", "endpoint", b"endpoint", "headers", b"headers", "limit", b"limit"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_limit", b"_limit"]) -> typing_extensions.Literal["limit"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_limit", b"_limit", "db", b"db", "limit", b"limit"
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_limit",
+ b"_limit",
+ "db",
+ b"db",
+ "endpoint",
+ b"endpoint",
+ "headers",
+ b"headers",
+ "limit",
+ b"limit",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_limit", b"_limit"]
+ ) -> typing_extensions.Literal["limit"] | None: ...
global___HttpPath = HttpPath
@typing_extensions.final
-class PreProcValue(google.protobuf.message.Message):
+class Eval(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
- @typing_extensions.final
- class Eval(google.protobuf.message.Message):
- DESCRIPTOR: google.protobuf.descriptor.Descriptor
+ SCHEMA_FIELD_NUMBER: builtins.int
+ EXPR_FIELD_NUMBER: builtins.int
+ PYCODE_FIELD_NUMBER: builtins.int
+ @property
+ def schema(self) -> schema_pb2.Schema: ...
+ @property
+ def expr(self) -> expression_pb2.Expr: ...
+ @property
+ def pycode(self) -> pycode_pb2.PyCode: ...
+ def __init__(
+ self,
+ *,
+ schema: schema_pb2.Schema | None = ...,
+ expr: expression_pb2.Expr | None = ...,
+ pycode: pycode_pb2.PyCode | None = ...,
+ ) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "eval_type",
+ b"eval_type",
+ "expr",
+ b"expr",
+ "pycode",
+ b"pycode",
+ "schema",
+ b"schema",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "eval_type",
+ b"eval_type",
+ "expr",
+ b"expr",
+ "pycode",
+ b"pycode",
+ "schema",
+ b"schema",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["eval_type", b"eval_type"]
+ ) -> typing_extensions.Literal["expr", "pycode"] | None: ...
- SCHEMA_FIELD_NUMBER: builtins.int
- EXPR_FIELD_NUMBER: builtins.int
- PYCODE_FIELD_NUMBER: builtins.int
- @property
- def schema(self) -> schema_pb2.Schema: ...
- @property
- def expr(self) -> expression_pb2.Expr: ...
- @property
- def pycode(self) -> pycode_pb2.PyCode: ...
- def __init__(
- self,
- *,
- schema: schema_pb2.Schema | None = ...,
- expr: expression_pb2.Expr | None = ...,
- pycode: pycode_pb2.PyCode | None = ...,
- ) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["eval_type", b"eval_type", "expr", b"expr", "pycode", b"pycode", "schema", b"schema"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["eval_type", b"eval_type", "expr", b"expr", "pycode", b"pycode", "schema", b"schema"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["eval_type", b"eval_type"]) -> typing_extensions.Literal["expr", "pycode"] | None: ...
+global___Eval = Eval
+
+@typing_extensions.final
+class PreProcValue(google.protobuf.message.Message):
+ DESCRIPTOR: google.protobuf.descriptor.Descriptor
REF_FIELD_NUMBER: builtins.int
VALUE_FIELD_NUMBER: builtins.int
@@ -1176,17 +2148,43 @@ class PreProcValue(google.protobuf.message.Message):
@property
def value(self) -> schema_pb2.Value: ...
@property
- def eval(self) -> global___PreProcValue.Eval: ...
+ def eval(self) -> global___Eval: ...
def __init__(
self,
*,
ref: builtins.str = ...,
value: schema_pb2.Value | None = ...,
- eval: global___PreProcValue.Eval | None = ...,
+ eval: global___Eval | None = ...,
+ ) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "eval",
+ b"eval",
+ "ref",
+ b"ref",
+ "value",
+ b"value",
+ "variant",
+ b"variant",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "eval",
+ b"eval",
+ "ref",
+ b"ref",
+ "value",
+ b"value",
+ "variant",
+ b"variant",
+ ],
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["eval", b"eval", "ref", b"ref", "value", b"value", "variant", b"variant"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["eval", b"eval", "ref", b"ref", "value", b"value", "variant", b"variant"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["variant", b"variant"]) -> typing_extensions.Literal["ref", "value", "eval"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["variant", b"variant"]
+ ) -> typing_extensions.Literal["ref", "value", "eval"] | None: ...
global___PreProcValue = PreProcValue
@@ -1205,8 +2203,15 @@ class MongoCollection(google.protobuf.message.Message):
db: global___ExtDatabase | None = ...,
collection_name: builtins.str = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["db", b"db"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["collection_name", b"collection_name", "db", b"db"]) -> None: ...
+ def HasField(
+ self, field_name: typing_extensions.Literal["db", b"db"]
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "collection_name", b"collection_name", "db", b"db"
+ ],
+ ) -> None: ...
global___MongoCollection = MongoCollection
@@ -1224,7 +2229,12 @@ class SnapshotData(google.protobuf.message.Message):
marker: builtins.str = ...,
num_retain: builtins.int = ...,
) -> None: ...
- def ClearField(self, field_name: typing_extensions.Literal["marker", b"marker", "num_retain", b"num_retain"]) -> None: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "marker", b"marker", "num_retain", b"num_retain"
+ ],
+ ) -> None: ...
global___SnapshotData = SnapshotData
@@ -1268,9 +2278,37 @@ class Style(google.protobuf.message.Message):
recreate: global___Recreate | None = ...,
snapshot: global___SnapshotData | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["Style", b"Style", "incremental", b"incremental", "recreate", b"recreate", "snapshot", b"snapshot"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["Style", b"Style", "incremental", b"incremental", "recreate", b"recreate", "snapshot", b"snapshot"]) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["Style", b"Style"]) -> typing_extensions.Literal["incremental", "recreate", "snapshot"] | None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "Style",
+ b"Style",
+ "incremental",
+ b"incremental",
+ "recreate",
+ b"recreate",
+ "snapshot",
+ b"snapshot",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "Style",
+ b"Style",
+ "incremental",
+ b"incremental",
+ "recreate",
+ b"recreate",
+ "snapshot",
+ b"snapshot",
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["Style", b"Style"]
+ ) -> (
+ typing_extensions.Literal["incremental", "recreate", "snapshot"] | None
+ ): ...
global___Style = Style
@@ -1298,8 +2336,15 @@ class Source(google.protobuf.message.Message):
key: builtins.str = ...,
value: global___PreProcValue | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["value", b"value"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+ def HasField(
+ self, field_name: typing_extensions.Literal["value", b"value"]
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "key", b"key", "value", b"value"
+ ],
+ ) -> None: ...
TABLE_FIELD_NUMBER: builtins.int
DATASET_FIELD_NUMBER: builtins.int
@@ -1317,6 +2362,8 @@ class Source(google.protobuf.message.Message):
UNTIL_FIELD_NUMBER: builtins.int
FILTER_FIELD_NUMBER: builtins.int
SAMPLING_STRATEGY_FIELD_NUMBER: builtins.int
+ FILTER_EXPR_FIELD_NUMBER: builtins.int
+ FILTER_SCHEMA_FIELD_NUMBER: builtins.int
@property
def table(self) -> global___ExtTable: ...
dataset: builtins.str
@@ -1331,7 +2378,11 @@ class Source(google.protobuf.message.Message):
@property
def starting_from(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
@property
- def pre_proc(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___PreProcValue]: ...
+ def pre_proc(
+ self,
+ ) -> google.protobuf.internal.containers.MessageMap[
+ builtins.str, global___PreProcValue
+ ]: ...
version: builtins.int
bounded: builtins.bool
@property
@@ -1342,6 +2393,12 @@ class Source(google.protobuf.message.Message):
def filter(self) -> pycode_pb2.PyCode: ...
@property
def sampling_strategy(self) -> global___SamplingStrategy: ...
+ @property
+ def filter_expr(self) -> expression_pb2.Expr: ...
+ @property
+ def filter_schema(self) -> schema_pb2.Schema:
+ """next id: 19"""
+
def __init__(
self,
*,
@@ -1354,24 +2411,141 @@ class Source(google.protobuf.message.Message):
timestamp_field: builtins.str = ...,
cdc: global___CDCStrategy.ValueType = ...,
starting_from: google.protobuf.timestamp_pb2.Timestamp | None = ...,
- pre_proc: collections.abc.Mapping[builtins.str, global___PreProcValue] | None = ...,
+ pre_proc: (
+ collections.abc.Mapping[builtins.str, global___PreProcValue] | None
+ ) = ...,
version: builtins.int = ...,
bounded: builtins.bool = ...,
idleness: google.protobuf.duration_pb2.Duration | None = ...,
until: google.protobuf.timestamp_pb2.Timestamp | None = ...,
filter: pycode_pb2.PyCode | None = ...,
sampling_strategy: global___SamplingStrategy | None = ...,
+ filter_expr: expression_pb2.Expr | None = ...,
+ filter_schema: schema_pb2.Schema | None = ...,
+ ) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_cursor",
+ b"_cursor",
+ "_filter",
+ b"_filter",
+ "_filter_expr",
+ b"_filter_expr",
+ "_filter_schema",
+ b"_filter_schema",
+ "_idleness",
+ b"_idleness",
+ "_sampling_strategy",
+ b"_sampling_strategy",
+ "cursor",
+ b"cursor",
+ "disorder",
+ b"disorder",
+ "every",
+ b"every",
+ "filter",
+ b"filter",
+ "filter_expr",
+ b"filter_expr",
+ "filter_schema",
+ b"filter_schema",
+ "idleness",
+ b"idleness",
+ "sampling_strategy",
+ b"sampling_strategy",
+ "starting_from",
+ b"starting_from",
+ "table",
+ b"table",
+ "until",
+ b"until",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_cursor",
+ b"_cursor",
+ "_filter",
+ b"_filter",
+ "_filter_expr",
+ b"_filter_expr",
+ "_filter_schema",
+ b"_filter_schema",
+ "_idleness",
+ b"_idleness",
+ "_sampling_strategy",
+ b"_sampling_strategy",
+ "bounded",
+ b"bounded",
+ "cdc",
+ b"cdc",
+ "cursor",
+ b"cursor",
+ "dataset",
+ b"dataset",
+ "disorder",
+ b"disorder",
+ "ds_version",
+ b"ds_version",
+ "every",
+ b"every",
+ "filter",
+ b"filter",
+ "filter_expr",
+ b"filter_expr",
+ "filter_schema",
+ b"filter_schema",
+ "idleness",
+ b"idleness",
+ "pre_proc",
+ b"pre_proc",
+ "sampling_strategy",
+ b"sampling_strategy",
+ "starting_from",
+ b"starting_from",
+ "table",
+ b"table",
+ "timestamp_field",
+ b"timestamp_field",
+ "until",
+ b"until",
+ "version",
+ b"version",
+ ],
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["_cursor", b"_cursor", "_filter", b"_filter", "_idleness", b"_idleness", "_sampling_strategy", b"_sampling_strategy", "cursor", b"cursor", "disorder", b"disorder", "every", b"every", "filter", b"filter", "idleness", b"idleness", "sampling_strategy", b"sampling_strategy", "starting_from", b"starting_from", "table", b"table", "until", b"until"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["_cursor", b"_cursor", "_filter", b"_filter", "_idleness", b"_idleness", "_sampling_strategy", b"_sampling_strategy", "bounded", b"bounded", "cdc", b"cdc", "cursor", b"cursor", "dataset", b"dataset", "disorder", b"disorder", "ds_version", b"ds_version", "every", b"every", "filter", b"filter", "idleness", b"idleness", "pre_proc", b"pre_proc", "sampling_strategy", b"sampling_strategy", "starting_from", b"starting_from", "table", b"table", "timestamp_field", b"timestamp_field", "until", b"until", "version", b"version"]) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_cursor", b"_cursor"]) -> typing_extensions.Literal["cursor"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_cursor", b"_cursor"]
+ ) -> typing_extensions.Literal["cursor"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_filter", b"_filter"]) -> typing_extensions.Literal["filter"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_filter", b"_filter"]
+ ) -> typing_extensions.Literal["filter"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_idleness", b"_idleness"]) -> typing_extensions.Literal["idleness"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal["_filter_expr", b"_filter_expr"],
+ ) -> typing_extensions.Literal["filter_expr"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_sampling_strategy", b"_sampling_strategy"]) -> typing_extensions.Literal["sampling_strategy"] | None: ...
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "_filter_schema", b"_filter_schema"
+ ],
+ ) -> typing_extensions.Literal["filter_schema"] | None: ...
+ @typing.overload
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_idleness", b"_idleness"]
+ ) -> typing_extensions.Literal["idleness"] | None: ...
+ @typing.overload
+ def WhichOneof(
+ self,
+ oneof_group: typing_extensions.Literal[
+ "_sampling_strategy", b"_sampling_strategy"
+ ],
+ ) -> typing_extensions.Literal["sampling_strategy"] | None: ...
global___Source = Source
@@ -1393,7 +2567,12 @@ class Sink(google.protobuf.message.Message):
key: builtins.str = ...,
value: builtins.str = ...,
) -> None: ...
- def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "key", b"key", "value", b"value"
+ ],
+ ) -> None: ...
TABLE_FIELD_NUMBER: builtins.int
DATASET_FIELD_NUMBER: builtins.int
@@ -1416,7 +2595,11 @@ class Sink(google.protobuf.message.Message):
def how(self) -> global___Style: ...
create: builtins.bool
@property
- def renames(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ...
+ def renames(
+ self,
+ ) -> google.protobuf.internal.containers.ScalarMap[
+ builtins.str, builtins.str
+ ]: ...
@property
def since(self) -> google.protobuf.timestamp_pb2.Timestamp: ...
@property
@@ -1431,19 +2614,85 @@ class Sink(google.protobuf.message.Message):
every: google.protobuf.duration_pb2.Duration | None = ...,
how: global___Style | None = ...,
create: builtins.bool = ...,
- renames: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
+ renames: (
+ collections.abc.Mapping[builtins.str, builtins.str] | None
+ ) = ...,
since: google.protobuf.timestamp_pb2.Timestamp | None = ...,
until: google.protobuf.timestamp_pb2.Timestamp | None = ...,
) -> None: ...
- def HasField(self, field_name: typing_extensions.Literal["_cdc", b"_cdc", "_how", b"_how", "_since", b"_since", "_until", b"_until", "cdc", b"cdc", "every", b"every", "how", b"how", "since", b"since", "table", b"table", "until", b"until"]) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["_cdc", b"_cdc", "_how", b"_how", "_since", b"_since", "_until", b"_until", "cdc", b"cdc", "create", b"create", "dataset", b"dataset", "ds_version", b"ds_version", "every", b"every", "how", b"how", "renames", b"renames", "since", b"since", "table", b"table", "until", b"until"]) -> None: ...
+ def HasField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_cdc",
+ b"_cdc",
+ "_how",
+ b"_how",
+ "_since",
+ b"_since",
+ "_until",
+ b"_until",
+ "cdc",
+ b"cdc",
+ "every",
+ b"every",
+ "how",
+ b"how",
+ "since",
+ b"since",
+ "table",
+ b"table",
+ "until",
+ b"until",
+ ],
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_cdc",
+ b"_cdc",
+ "_how",
+ b"_how",
+ "_since",
+ b"_since",
+ "_until",
+ b"_until",
+ "cdc",
+ b"cdc",
+ "create",
+ b"create",
+ "dataset",
+ b"dataset",
+ "ds_version",
+ b"ds_version",
+ "every",
+ b"every",
+ "how",
+ b"how",
+ "renames",
+ b"renames",
+ "since",
+ b"since",
+ "table",
+ b"table",
+ "until",
+ b"until",
+ ],
+ ) -> None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_cdc", b"_cdc"]) -> typing_extensions.Literal["cdc"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_cdc", b"_cdc"]
+ ) -> typing_extensions.Literal["cdc"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_how", b"_how"]) -> typing_extensions.Literal["how"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_how", b"_how"]
+ ) -> typing_extensions.Literal["how"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_since", b"_since"]) -> typing_extensions.Literal["since"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_since", b"_since"]
+ ) -> typing_extensions.Literal["since"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["_until", b"_until"]) -> typing_extensions.Literal["until"] | None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_until", b"_until"]
+ ) -> typing_extensions.Literal["until"] | None: ...
global___Sink = Sink
diff --git a/fennel/internal_lib/to_proto/to_proto.py b/fennel/internal_lib/to_proto/to_proto.py
index 4f1647da5..c9b68e6c9 100644
--- a/fennel/internal_lib/to_proto/to_proto.py
+++ b/fennel/internal_lib/to_proto/to_proto.py
@@ -26,6 +26,7 @@
import fennel.gen.secret_pb2 as secret_proto
import fennel.gen.dataset_pb2 as ds_proto
import fennel.gen.expectations_pb2 as exp_proto
+import fennel.gen.expression_pb2 as expression_proto
import fennel.gen.featureset_pb2 as fs_proto
import fennel.gen.http_auth_pb2 as http_auth_proto
import fennel.gen.kinesis_pb2 as kinesis_proto
@@ -41,6 +42,7 @@
Sample,
PreProcValue,
Certificate,
+ WhereValue,
)
from fennel.datasets import Dataset, Pipeline, Field
from fennel.datasets.datasets import (
@@ -498,6 +500,43 @@ def _validate_source_pre_proc(
)
+def _validate_source_where(where: WhereValue, ds: Dataset):
+ if isinstance(where, Callable): # type: ignore
+ return
+ elif isinstance(where, connectors.Eval):
+ # Run output col dtype validation for Expr.
+ if isinstance(where.eval_type, (Expr, TypedExpr)):
+ expr = (
+ where.eval_type
+ if isinstance(where.eval_type, Expr)
+ else where.eval_type.expr
+ )
+ typed_dtype = (
+ None
+ if isinstance(where.eval_type, Expr)
+ else where.eval_type.dtype
+ )
+ input_schema = ds.schema()
+ if where.additional_schema:
+ for name, dtype in where.additional_schema.items():
+ input_schema[name] = dtype
+ expr_type = expr.typeof(input_schema)
+ if expr_type != bool:
+ printer = ExprPrinter()
+ raise TypeError(
+ f"expression in source where is of type `{dtype_to_string(expr_type)}` in Dataset `{ds._name}`, "
+ f"can not be cast to bool. Full expression: `{printer.print(expr.root)}`"
+ )
+ if typed_dtype and expr_type != typed_dtype:
+ printer = ExprPrinter()
+ raise TypeError(
+ f"expression in source where in Dataset `{ds._name}` can not be cast to `{dtype_to_string(typed_dtype)}`, "
+ f"evaluated dtype is `{dtype_to_string(expr_type)}`. Full expression: `{printer.print(expr.root)}`"
+ )
+ else:
+ return
+
+
def source_from_ds(
ds: Dataset, env: Optional[str] = None
) -> List[connectors.DataConnector]:
@@ -526,6 +565,8 @@ def source_proto_from_ds(
for src in sources:
if src.pre_proc:
_validate_source_pre_proc(src.pre_proc, ds)
+ if src.where:
+ _validate_source_where(src.where, ds)
ret.append(_conn_to_source_proto(src, ds))
return ret # type: ignore
@@ -757,6 +798,19 @@ def _to_field_lookup_proto(
# ------------------------------------------------------------------------------
+def _preproc_assign_to_pycode(
+ dataset: Dataset, column_name: str, func: Callable
+) -> pycode_proto.PyCode:
+ return wrap_function(
+ dataset._name,
+ get_dataset_core_code(dataset),
+ "",
+ to_includes_proto(func),
+ is_assign=True,
+ column_name=column_name,
+ )
+
+
def _pre_proc_value_to_proto(
dataset: Dataset,
column_name: str,
@@ -783,14 +837,14 @@ def _pre_proc_value_to_proto(
else:
root = pre_proc_value.eval_type.expr.root
return connector_proto.PreProcValue(
- eval=connector_proto.PreProcValue.Eval(
+ eval=connector_proto.Eval(
schema=schema,
expr=serializer.serialize(root),
)
)
else:
return connector_proto.PreProcValue(
- eval=connector_proto.PreProcValue.Eval(
+ eval=connector_proto.Eval(
schema=schema,
pycode=_preproc_assign_to_pycode(
dataset, column_name, pre_proc_value.eval_type
@@ -850,11 +904,9 @@ def _pre_proc_to_proto(
return proto_pre_proc
-def _preproc_filter_to_pycode(
- dataset: Dataset, func: Optional[Callable]
+def _where_filter_to_pycode(
+ dataset: Dataset, func: Callable
) -> Optional[pycode_proto.PyCode]:
- if func is None:
- return None
return wrap_function(
dataset._name,
get_dataset_core_code(dataset),
@@ -864,17 +916,46 @@ def _preproc_filter_to_pycode(
)
-def _preproc_assign_to_pycode(
- dataset: Dataset, column_name: str, func: Callable
-) -> pycode_proto.PyCode:
- return wrap_function(
- dataset._name,
- get_dataset_core_code(dataset),
- "",
- to_includes_proto(func),
- is_assign=True,
- column_name=column_name,
- )
+def _where_value_to_proto(
+ dataset: Dataset, where: Optional[connectors.WhereValue]
+) -> Tuple[
+ Optional[pycode_proto.PyCode],
+ Optional[expression_proto.Expr],
+ Optional[schema_proto.Schema],
+]:
+ if where is None:
+ return None, None, None
+ elif isinstance(where, Callable): # type: ignore
+ return _where_filter_to_pycode(dataset, where), None, None # type: ignore
+ else:
+ assert isinstance(where, connectors.Eval)
+
+ schema = None
+ if where.additional_schema:
+ fields = [
+ schema_proto.Field(name=name, dtype=get_datatype(dtype))
+ for (name, dtype) in where.additional_schema.items()
+ ]
+ schema = schema_proto.Schema(fields=fields)
+
+ if isinstance(where.eval_type, (Expr, TypedExpr)):
+ serializer = ExprSerializer()
+ if isinstance(where.eval_type, Expr):
+ root = where.eval_type.root
+ else:
+ root = where.eval_type.expr.root
+ return (
+ None,
+ serializer.serialize(root),
+ schema,
+ )
+ else:
+ assert isinstance(where.eval_type, Callable) # type: ignore
+ return (
+ _where_filter_to_pycode(dataset, where.eval_type),
+ None,
+ schema,
+ )
def _conn_to_source_proto(
@@ -928,6 +1009,9 @@ def _webhook_to_source_proto(
retention=to_duration_proto(data_source.retention),
),
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
if not connector.cdc:
raise ValueError("CDC should always be set for webhook source")
return (
@@ -954,7 +1038,9 @@ def _webhook_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -976,6 +1062,9 @@ def _kafka_conn_to_source_proto(
data_source.sasl_plain_username,
data_source.sasl_plain_password,
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
source = connector_proto.Source(
table=connector_proto.ExtTable(
kafka_topic=connector_proto.KafkaTopic(
@@ -1000,7 +1089,9 @@ def _kafka_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
)
return (ext_db, source)
@@ -1150,6 +1241,9 @@ def _s3_conn_to_source_proto(
presorted=connector.presorted,
spread=connector.spread,
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
source = connector_proto.Source(
table=ext_table,
dataset=dataset._name,
@@ -1171,7 +1265,9 @@ def _s3_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
)
return (ext_db, source)
@@ -1290,7 +1386,6 @@ def _how_to_proto(
)
raise ValueError("Invalid value for how to convert to proto {}".format(how))
- ...
def _s3_conn_to_sink_proto(
@@ -1449,6 +1544,9 @@ def _bigquery_conn_to_source_proto(
ext_db,
table_name=connector.table_name,
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
return (
ext_db,
connector_proto.Source(
@@ -1472,7 +1570,9 @@ def _bigquery_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -1529,6 +1629,9 @@ def _redshift_conn_to_source_proto(
db=ext_db,
table_name=connector.table_name,
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
return (
ext_db,
connector_proto.Source(
@@ -1552,7 +1655,9 @@ def _redshift_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -1633,6 +1738,9 @@ def _mongo_conn_to_source_proto(
db=ext_db,
table_name=connector.table_name,
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
return (
ext_db,
connector_proto.Source(
@@ -1656,7 +1764,9 @@ def _mongo_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -1711,6 +1821,9 @@ def _pubsub_conn_to_source_proto(
service_account_key_secret=to_secret_proto(service_account_key),
),
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
return (
ext_db,
connector_proto.Source(
@@ -1735,7 +1848,9 @@ def _pubsub_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -1802,6 +1917,9 @@ def _snowflake_conn_to_source_proto(
ext_table = _snowflake_to_ext_table_proto(
db=ext_db, table_name=connector.table_name
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
return (
ext_db,
connector_proto.Source(
@@ -1825,7 +1943,9 @@ def _snowflake_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -1913,6 +2033,9 @@ def _mysql_conn_to_source_proto(
ext_table = _mysql_to_ext_table_proto(
db=ext_db, table_name=connector.table_name
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
return (
ext_db,
connector_proto.Source(
@@ -1936,7 +2059,9 @@ def _mysql_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -2011,6 +2136,9 @@ def _pg_conn_to_source_proto(
ext_table = _pg_to_ext_table_proto(
db=ext_db, table_name=connector.table_name
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
return (
ext_db,
connector_proto.Source(
@@ -2034,7 +2162,9 @@ def _pg_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -2106,6 +2236,9 @@ def _kinesis_conn_to_source_proto(
init_position=connector.init_position,
format=connector.format,
)
+ filter_proto, filter_expr, filter_schema = _where_value_to_proto(
+ dataset, connector.where
+ )
return (
ext_db,
connector_proto.Source(
@@ -2126,7 +2259,9 @@ def _kinesis_conn_to_source_proto(
if connector.idleness
else None
),
- filter=_preproc_filter_to_pycode(dataset, connector.where),
+ filter=filter_proto,
+ filter_expr=filter_expr,
+ filter_schema=filter_schema,
),
)
@@ -2298,7 +2433,7 @@ def to_extractor_pycode(
extractor: Extractor,
featureset: Featureset,
fs_obj_map: Dict[str, Featureset],
-) -> pycode_proto.PyCode:
+) -> Optional[pycode_proto.PyCode]:
if extractor.extractor_type != ExtractorType.PY_FUNC:
return None
if not extractor.func:
diff --git a/fennel/testing/data_engine.py b/fennel/testing/data_engine.py
index e2ffde8bc..b619433ff 100644
--- a/fennel/testing/data_engine.py
+++ b/fennel/testing/data_engine.py
@@ -39,7 +39,11 @@
from fennel.gen import schema_pb2 as schema_proto
from fennel.gen.dataset_pb2 import CoreDataset
from fennel.internal_lib.duration import Duration, duration_to_timedelta
-from fennel.internal_lib.schema import data_schema_check, get_datatype
+from fennel.internal_lib.schema import (
+ data_schema_check,
+ get_datatype,
+ get_python_type_from_pd,
+)
from fennel.internal_lib.to_proto import (
dataset_to_proto,
get_dataset_core_code,
@@ -172,27 +176,60 @@ def _preproc_df(
return new_df
-def _preproc_where_df(
- df: pd.DataFrame, filter_fn: Callable, ds: Dataset
-) -> pd.DataFrame:
+def _preproc_callable_df(df: pd.DataFrame, callable: Callable, ds: Dataset):
filter_func_pycode = wrap_function(
ds._name,
get_dataset_core_code(ds),
"",
- to_includes_proto(filter_fn),
+ to_includes_proto(callable),
is_filter=True,
)
- new_df = df.copy()
mod = types.ModuleType(filter_func_pycode.entry_point)
code = filter_func_pycode.imports + "\n" + filter_func_pycode.generated_code
exec(code, mod.__dict__)
func = mod.__dict__[filter_func_pycode.entry_point]
try:
- new_df = func(new_df)
+ df = func(df)
except Exception as e:
raise Exception(
- f"Error in filter function `{filter_fn.__name__}` for preproc , {e}"
+ f"Error in filter function `{callable.__name__}` for preproc , {e}"
)
+ return df
+
+
+def _preproc_where_df(
+ df: pd.DataFrame, where_val: connectors.WhereValue, ds: Dataset
+) -> pd.DataFrame:
+ new_df = df.copy()
+ schema = ds.schema()
+
+ if isinstance(where_val, Callable): # type: ignore
+ new_df = _preproc_callable_df(new_df, where_val, ds) # type: ignore
+ else:
+ assert isinstance(where_val, connectors.Eval)
+ if where_val.additional_schema:
+ for name, dtype in where_val.additional_schema.items():
+ if name not in df.columns:
+ raise ValueError(
+ f"Field `{name}` defined in schema for eval where not found "
+ f"in dataframe."
+ )
+ if dtype != get_python_type_from_pd(schema[name]):
+ raise ValueError(
+ f"Field `{name}` defined in schema for eval where has "
+ f"different type in the dataframe."
+ )
+
+ if isinstance(where_val.eval_type, Expr):
+ filtered = where_val.eval_type.eval(new_df, schema)
+ new_df = new_df.iloc[filtered.values.tolist()]
+ elif isinstance(where_val.eval_type, TypedExpr):
+ filtered = where_val.eval_type.expr.eval(new_df, schema)
+ new_df = new_df.iloc[filtered.values.tolist()]
+ else:
+ assert isinstance(where_val.eval_type, Callable) # type: ignore
+ new_df = _preproc_callable_df(new_df, where_val.eval_type, ds)
+
return new_df
@@ -203,7 +240,7 @@ class _SrcInfo:
bounded: bool
idleness: Optional[Duration]
prev_log_time: Optional[datetime] = None
- where: Optional[Callable] = None
+ where: Optional[connectors.WhereValue] = None
@dataclass
@@ -410,14 +447,14 @@ def log(
if len(wheres) > 0 and wheres[0] is not None:
assert (
len(wheres) == 1
- ), f"Multiple preproc wheres found for {ds} and {webhook_endpoint}"
+ ), f"Multiple wheres found for {ds} and {webhook_endpoint}"
try:
df = _preproc_where_df(
df, wheres[0], self.datasets[ds].dataset
)
except ValueError as e:
raise ValueError(
- f"Error using filter pre_proc for dataset `{ds}`: {str(e)}",
+ f"Error using where for dataset `{ds}`: {str(e)}",
)
# Cast output to pyarrow dtypes
df = cast_df_to_schema(df, schema)
diff --git a/pyproject.toml b/pyproject.toml
index 43e562754..a6244732a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "fennel-ai"
-version = "1.5.51"
+version = "1.5.52"
description = "The modern realtime feature engineering platform"
authors = ["Fennel AI "]
packages = [{ include = "fennel" }]