spec.json

{
  "documentationUrl": "https://docs.airbyte.com/integrations/sources/kafka",
  "supportsIncremental": true,
  "supported_source_sync_modes": ["append"],
  "connectionSpecification": {
    "$schema": "http://json-schema.org/draft-07/schema#",
    "title": "Kafka Source Spec",
    "type": "object",
    "required": ["bootstrap_servers", "subscription", "protocol"],
    "additionalProperties": true,
    "properties": {
      "MessageFormat": {
        "title": "MessageFormat",
        "type": "object",
        "description": "The serialization used based on this ",
        "oneOf": [
          {
            "title": "JSON",
            "properties": {
              "deserialization_type": {
                "type": "string",
                "const": "JSON"
              }
            }
          },
          {
            "title": "AVRO",
            "properties": {
              "deserialization_type": {
                "const": "AVRO"
              },
              "deserialization_strategy": {
                "type": "string",
                "enum": [
                  "TopicNameStrategy",
                  "RecordNameStrategy",
                  "TopicRecordNameStrategy"
                ],
                "default": "TopicNameStrategy"
              },
              "schema_registry_url": {
                "type": "string",
                "examples": ["http://localhost:8081"]
              },
              "schema_registry_username": {
                "type": "string",
                "default": ""
              },
              "schema_registry_password": {
                "type": "string",
                "default": "",
                "airbyte_secret": true
              }
            }
          }
        ]
      },
      "bootstrap_servers": {
        "title": "Bootstrap Servers",
        "description": "A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. The client will make use of all servers irrespective of which servers are specified here for bootstrapping&mdash;this list only impacts the initial hosts used to discover the full set of servers. This list should be in the form <code>host1:port1,host2:port2,...</code>. Since these servers are just used for the initial connection to discover the full cluster membership (which may change dynamically), this list need not contain the full set of servers (you may want more than one, though, in case a server is down).",
        "type": "string",
        "examples": ["kafka-broker1:9092,kafka-broker2:9092"]
      },
      "subscription": {
        "title": "Subscription Method",
        "type": "object",
        "description": "You can choose to manually assign a list of partitions, or subscribe to all topics matching specified pattern to get dynamically assigned partitions.",
        "oneOf": [
          {
            "title": "Manually assign a list of partitions",
            "required": ["subscription_type", "topic_partitions"],
            "properties": {
              "subscription_type": {
                "description": "Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment and will replace the previous assignment (if there is one).\nIf the given list of topic partitions is empty, it is treated the same as unsubscribe().",
                "type": "string",
                "const": "assign"
              },
              "topic_partitions": {
                "title": "List of topic:partition Pairs",
                "type": "string",
                "examples": ["sample.topic:0, sample.topic:1"]
              }
            }
          },
          {
            "title": "Subscribe to all topics matching specified pattern",
            "required": ["subscription_type", "topic_pattern"],
            "properties": {
              "subscription_type": {
                "description": "The Topic pattern from which the records will be read.",
                "type": "string",
                "const": "subscribe"
              },
              "topic_pattern": {
                "title": "Topic Pattern",
                "type": "string",
                "examples": ["sample.topic"]
              }
            }
          }
        ]
      },
      "test_topic": {
        "title": "Test Topic",
        "description": "The Topic to test in case the Airbyte can consume messages.",
        "type": "string",
        "examples": ["test.topic"]
      },
      "group_id": {
        "title": "Group ID",
        "description": "The Group ID is how you distinguish different consumer groups.",
        "type": "string",
        "examples": ["group.id"]
      },
      "max_poll_records": {
        "title": "Max Poll Records",
        "description": "The maximum number of records returned in a single call to poll(). Note, that max_poll_records does not impact the underlying fetching behavior. The consumer will cache the records from each fetch request and returns them incrementally from each poll.",
        "type": "integer",
        "default": 500
      },
      "polling_time": {
        "title": "Polling Time",
        "description": "Amount of time in milliseconds Kafka connector should try to poll for messages.",
        "type": "integer",
        "default": 100
      },
      "protocol": {
        "title": "Protocol",
        "type": "object",
        "description": "The Protocol used to communicate with brokers.",
        "oneOf": [
          {
            "title": "PLAINTEXT",
            "required": ["security_protocol"],
            "properties": {
              "security_protocol": {
                "type": "string",
                "const": "PLAINTEXT"
              }
            }
          },
          {
            "title": "SASL PLAINTEXT",
            "required": [
              "security_protocol",
              "sasl_mechanism",
              "sasl_jaas_config"
            ],
            "properties": {
              "security_protocol": {
                "type": "string",
                "const": "SASL_PLAINTEXT"
              },
              "sasl_mechanism": {
                "title": "SASL Mechanism",
                "description": "The SASL mechanism used for client connections. This may be any mechanism for which a security provider is available.",
                "type": "string",
                "const": "PLAIN"
              },
              "sasl_jaas_config": {
                "title": "SASL JAAS Config",
                "description": "The JAAS login context parameters for SASL connections in the format used by JAAS configuration files.",
                "type": "string",
                "default": "",
                "airbyte_secret": true
              }
            }
          },
          {
            "title": "SASL SSL",
            "required": [
              "security_protocol",
              "sasl_mechanism",
              "sasl_jaas_config"
            ],
            "properties": {
              "security_protocol": {
                "type": "string",
                "const": "SASL_SSL"
              },
              "sasl_mechanism": {
                "title": "SASL Mechanism",
                "description": "The SASL mechanism used for client connections. This may be any mechanism for which a security provider is available.",
                "type": "string",
                "default": "GSSAPI",
                "enum": [
                  "GSSAPI",
                  "OAUTHBEARER",
                  "SCRAM-SHA-256",
                  "SCRAM-SHA-512",
                  "PLAIN"
                ]
              },
              "sasl_jaas_config": {
                "title": "SASL JAAS Config",
                "description": "The JAAS login context parameters for SASL connections in the format used by JAAS configuration files.",
                "type": "string",
                "default": "",
                "airbyte_secret": true
              },
              "oauthbearer_token_endpoint_url": {
                "title": "OAUTHBEARER token endpoint URL",
                "type": "string",
                "description": "Endpoint for requesting tokens if setting SASL Mechanism to OAUTHBEARER",
                "default": ""
              }
            }
          }
        ]
      },
      "client_id": {
        "title": "Client ID",
        "description": "An ID string to pass to the server when making requests. The purpose of this is to be able to track the source of requests beyond just ip/port by allowing a logical application name to be included in server-side request logging.",
        "type": "string",
        "examples": ["airbyte-consumer"]
      },
      "enable_auto_commit": {
        "title": "Enable Auto Commit",
        "description": "If true, the consumer's offset will be periodically committed in the background.",
        "type": "boolean",
        "default": true
      },
      "auto_commit_interval_ms": {
        "title": "Auto Commit Interval, ms",
        "description": "The frequency in milliseconds that the consumer offsets are auto-committed to Kafka if enable.auto.commit is set to true.",
        "type": "integer",
        "default": 5000
      },
      "client_dns_lookup": {
        "title": "Client DNS Lookup",
        "description": "Controls how the client uses DNS lookups. If set to use_all_dns_ips, connect to each returned IP address in sequence until a successful connection is established. After a disconnection, the next IP is used. Once all IPs have been used once, the client resolves the IP(s) from the hostname again. If set to resolve_canonical_bootstrap_servers_only, resolve each bootstrap address into a list of canonical names. After the bootstrap phase, this behaves the same as use_all_dns_ips. If set to default (deprecated), attempt to connect to the first IP address returned by the lookup, even if the lookup returns multiple IP addresses.",
        "type": "string",
        "default": "use_all_dns_ips",
        "enum": [
          "default",
          "use_all_dns_ips",
          "resolve_canonical_bootstrap_servers_only"
        ]
      },
      "retry_backoff_ms": {
        "title": "Retry Backoff, ms",
        "description": "The amount of time to wait before attempting to retry a failed request to a given topic partition. This avoids repeatedly sending requests in a tight loop under some failure scenarios.",
        "type": "integer",
        "default": 100
      },
      "request_timeout_ms": {
        "title": "Request Timeout, ms",
        "description": "The configuration controls the maximum amount of time the client will wait for the response of a request. If the response is not received before the timeout elapses the client will resend the request if necessary or fail the request if retries are exhausted.",
        "type": "integer",
        "default": 30000
      },
      "receive_buffer_bytes": {
        "title": "Receive Buffer, bytes",
        "description": "The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. If the value is -1, the OS default will be used.",
        "type": "integer",
        "default": 32768
      },
      "auto_offset_reset": {
        "title": "Auto Offset Reset",
        "description": "What to do when there is no initial offset in Kafka or if the current offset does not exist any more on the server - earliest: automatically reset the offset to the earliest offset, latest: automatically reset the offset to the latest offset, none: throw exception to the consumer if no previous offset is found for the consumer's group, anything else: throw exception to the consumer.",
        "type": "string",
        "default": "latest",
        "enum": ["latest", "earliest", "none"]
      },
      "repeated_calls": {
        "title": "Repeated Calls",
        "description": "The number of repeated calls to poll() if no messages were received.",
        "type": "integer",
        "default": 3
      },
      "max_records_process": {
        "title": "Maximum Records",
        "description": "The Maximum to be processed per execution",
        "type": "integer",
        "default": 100000
      }
    }
  }
}