[Data] Add Kafka as datasource

doc/source/data/api/input_output.rst

@@ -368,6 +368,15 @@ WebDataset
 
 .. _data_source_api:
 
+Kafka
+-----
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_kafka
+
 Datasource API
 --------------
 

python/ray/data/BUILD.bazel

@@ -1129,6 +1129,20 @@ py_test(
     ],
 )
 
+py_test(
+    name = "test_kafka",
+    size = "medium",
+    srcs = ["tests/test_kafka.py"],
+    tags = [
+        "exclusive",
+        "team:data",
+    ],
+    deps = [
+        ":conftest",
+        "//:ray_lib",
+    ],
+)
+
 py_test(
     name = "test_metadata_provider",
     size = "small",

python/ray/data/__init__.py

@@ -27,6 +27,7 @@
 from ray.data.iterator import DataIterator, DatasetIterator
 from ray.data.preprocessor import Preprocessor
 from ray.data.read_api import (  # noqa: F401
+    KafkaAuthConfig,
     from_arrow,
     from_arrow_refs,
     from_blocks,
@@ -55,6 +56,7 @@
     read_datasource,
     read_delta,
     read_delta_sharing_tables,
+    read_kafka,
     read_hudi,
     read_iceberg,
     read_images,
@@ -162,6 +164,8 @@
     "read_datasource",
     "read_delta",
     "read_delta_sharing_tables",
+    "read_kafka",
+    "KafkaAuthConfig",
     "read_hudi",
     "read_iceberg",
     "read_images",