airbyte_protocol.yaml

---
"$schema": http://json-schema.org/draft-07/schema#
"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml
title: AirbyteProtocol
type: object
description: AirbyteProtocol structs
version: 1.0.0
properties:
  airbyte_message:
    "$ref": "#/definitions/AirbyteMessage"
  configured_airbyte_catalog:
    "$ref": "#/definitions/ConfiguredAirbyteCatalog"
definitions:
  AirbyteMessage:
    type: object
    additionalProperties: true
    required:
      - type
    properties:
      type:
        description: "Message type"
        type: string
        enum:
          - RECORD
          - STATE
          - LOG
          - SPEC
          - CONNECTION_STATUS
          - CATALOG
          - TRACE
          - CONTROL
      log:
        description: "log message: any kind of logging you want the platform to know about."
        "$ref": "#/definitions/AirbyteLogMessage"
      spec:
        "$ref": "#/definitions/ConnectorSpecification"
      # todo (cgardens) - prefer snake case for field names.
      connectionStatus:
        "$ref": "#/definitions/AirbyteConnectionStatus"
      catalog:
        description: "catalog message: the catalog"
        "$ref": "#/definitions/AirbyteCatalog"
      record:
        description: "record message: the record"
        "$ref": "#/definitions/AirbyteRecordMessage"
      state:
        description: "schema message: the state. Must be the last message produced. The platform uses this information"
        "$ref": "#/definitions/AirbyteStateMessage"
      trace:
        description: "trace message: a message to communicate information about the status and performance of a connector"
        "$ref": "#/definitions/AirbyteTraceMessage"
      control:
        description: "connector config message: a message to communicate an updated configuration from a connector that should be persisted"
        "$ref": "#/definitions/AirbyteControlMessage"
  AirbyteRecordMessage:
    type: object
    additionalProperties: true
    required:
      - stream
      - data
      - emitted_at
    properties:
      namespace:
        description: "namespace the data is associated with"
        type: string
      stream:
        description: "stream the data is associated with"
        type: string
      data:
        description: "record data"
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      emitted_at:
        description: "when the data was emitted from the source. epoch in millisecond."
        type: integer
  AirbyteStateMessage:
    type: object
    additionalProperties: true
    properties:
      type:
        "$ref": "#/definitions/AirbyteStateType"
      stream:
        "$ref": "#/definitions/AirbyteStreamState"
      global:
        "$ref": "#/definitions/AirbyteGlobalState"
      data:
        description: "(Deprecated) the state data"
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
  AirbyteStateType:
    type: string
    description: >
      The type of state the other fields represent.
      Is set to LEGACY, the state data should be read from the `data` field for backwards compatibility.
      If not set, assume the state object is type LEGACY.
      GLOBAL means that the state should be read from `global` and means that it represents the state for all the streams. It contains one shared
      state and individual stream states.
      PER_STREAM means that the state should be read from `stream`. The state present in this field correspond to the isolated state of the
      associated stream description.
    enum:
      - GLOBAL
      - STREAM
      - LEGACY
  AirbyteStreamState:
    type: object
    additionalProperties: true
    required:
      - stream_descriptor
    properties:
      stream_descriptor:
        "$ref": "#/definitions/StreamDescriptor"
      stream_state:
        "$ref": "#/definitions/AirbyteStateBlob"
  AirbyteGlobalState:
    type: object
    additionalProperties: true
    required:
      - stream_states
    properties:
      shared_state:
        "$ref": "#/definitions/AirbyteStateBlob"
      stream_states:
        type: array
        items:
          "$ref": "#/definitions/AirbyteStreamState"
  StreamDescriptor:
    type: object
    additionalProperties: true
    required:
      - name
    properties:
      name:
        type: string
      namespace:
        type: string
  AirbyteStateBlob:
    type: object
    additionalProperties: true
    description: "the state data"
    existingJavaType: com.fasterxml.jackson.databind.JsonNode
  AirbyteLogMessage:
    type: object
    additionalProperties: true
    required:
      - level
      - message
    properties:
      level:
        description: "log level"
        type: string
        enum:
          - FATAL
          - ERROR
          - WARN
          - INFO
          - DEBUG
          - TRACE
      message:
        description: "log message"
        type: string
      stack_trace:
        description: "an optional stack trace if the log message corresponds to an exception"
        type: string
  AirbyteTraceMessage:
    type: object
    additionalProperties: true
    required:
      - type
      - emitted_at
    properties:
      type:
        title: "trace type" # this title is required to avoid python codegen conflicts with the "type" parameter in AirbyteMessage. See https://github.com/airbytehq/airbyte/pull/12581
        description: "the type of trace message"
        type: string
        enum:
          - ERROR
          - ESTIMATE
      emitted_at:
        description: "the time in ms that the message was emitted"
        type: number
      error:
        description: "error trace message: the error object"
        "$ref": "#/definitions/AirbyteErrorTraceMessage"
      estimate:
        description: "Estimate trace message: a guess at how much data will be produced in this sync"
        "$ref": "#/definitions/AirbyteEstimateTraceMessage"
  AirbyteErrorTraceMessage:
    type: object
    additionalProperties: true
    required:
      - message
    properties:
      message:
        description: A user-friendly message that indicates the cause of the error
        type: string
      internal_message:
        description: The internal error that caused the failure
        type: string
      stack_trace:
        description: The full stack trace of the error
        type: string
      failure_type:
        description: The type of error
        type: string
        enum:
          - system_error
          - config_error
  AirbyteEstimateTraceMessage:
    type: object
    additionalProperties: true
    required:
      - name
      - type
    properties:
      name:
        description: The name of the stream
        type: string
      type:
        title: "estimate type" # this title is required to avoid python codegen conflicts with the "type" parameter in AirbyteMessage. See https://github.com/airbytehq/airbyte/pull/12581
        description: >
          Estimates are either per-stream (STREAM) or for the entire sync (SYNC).
          STREAM is preferred, and requires the source to count how many records are about to be emitted per-stream (e.g. there will be 100 rows from this table emitted).
          For the rare source which cannot tell which stream a record belongs to before reading (e.g. CDC databases), SYNC estimates can be emitted.
          Sources should not emit both STREAM and SOURCE estimates within a sync.
        type: string
        enum:
          - STREAM
          - SYNC
      namespace:
        description: The namespace of the stream
        type: string
      row_estimate:
        description: The estimated number of rows to be emitted by this sync for this stream
        type: integer
      byte_estimate:
        description: The estimated number of bytes to be emitted by this sync for this stream
        type: integer
  AirbyteControlMessage:
    type: object
    additionalProperties: true
    required:
      - type
      - emitted_at
    properties:
      type:
        title: orchestrator type
        description: "the type of orchestrator message"
        type: string
        enum:
          - CONNECTOR_CONFIG
      emitted_at:
        description: "the time in ms that the message was emitted"
        type: number
      connectorConfig:
        description: "connector config orchestrator message: the updated config for the platform to store for this connector"
        "$ref": "#/definitions/AirbyteControlConnectorConfigMessage"
  AirbyteControlConnectorConfigMessage:
    type: object
    additionalProperties: true
    required:
      - config
    properties:
      config:
        description: "the config items from this connector's spec to update"
        type: object
        additionalProperties: true
  AirbyteConnectionStatus:
    type: object
    description: Airbyte connection status
    additionalProperties: true
    required:
      - status
    properties:
      status:
        type: string
        enum:
          - SUCCEEDED
          - FAILED
      message:
        type: string
  AirbyteCatalog:
    type: object
    additionalProperties: true
    description: Airbyte stream schema catalog
    required:
      - streams
    properties:
      streams:
        type: array
        items:
          "$ref": "#/definitions/AirbyteStream"
  AirbyteStream:
    type: object
    additionalProperties: true
    required:
      - name
      - json_schema
      - supported_sync_modes
    properties:
      name:
        type: string
        description: Stream's name.
      json_schema:
        description: Stream schema using Json Schema specs.
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      supported_sync_modes:
        description: List of sync modes supported by this stream.
        type: array
        minItems: 1
        items:
          "$ref": "#/definitions/SyncMode"
      source_defined_cursor:
        description: If the source defines the cursor field, then any other cursor field inputs will be ignored. If it does not, either the user_provided one is used, or the default one is used as a backup.
        type: boolean
      default_cursor_field:
        description: Path to the field that will be used to determine if a record is new or modified since the last sync. If not provided by the source, the end user will have to specify the comparable themselves.
        type: array
        items:
          type: string
      source_defined_primary_key:
        description: If the source defines the primary key, paths to the fields that will be used as a primary key. If not provided by the source, the end user will have to specify the primary key themselves.
        type: array
        items:
          type: array
          items:
            type: string
      namespace:
        description: Optional Source-defined namespace. Currently only used by JDBC destinations to determine what schema to write to. Airbyte streams from the same sources should have the same namespace.
        type: string
  ConfiguredAirbyteCatalog:
    type: object
    additionalProperties: true
    description: Airbyte stream schema catalog
    required:
      - streams
    properties:
      streams:
        type: array
        items:
          "$ref": "#/definitions/ConfiguredAirbyteStream"
  ConfiguredAirbyteStream:
    type: object
    additionalProperties: true
    required:
      - stream
      - sync_mode
      - destination_sync_mode
    properties:
      stream:
        "$ref": "#/definitions/AirbyteStream"
      sync_mode:
        "$ref": "#/definitions/SyncMode"
        default: full_refresh
      cursor_field:
        description: Path to the field that will be used to determine if a record is new or modified since the last sync. This field is REQUIRED if `sync_mode` is `incremental`. Otherwise it is ignored.
        type: array
        items:
          type: string
      destination_sync_mode:
        "$ref": "#/definitions/DestinationSyncMode"
        default: append
      primary_key:
        description: Paths to the fields that will be used as primary key. This field is REQUIRED if `destination_sync_mode` is `*_dedup`. Otherwise it is ignored.
        type: array
        items:
          type: array
          items:
            type: string
  SyncMode:
    type: string
    enum:
      - full_refresh
      - incremental
  DestinationSyncMode:
    type: string
    enum:
      - append
      - overwrite
      #- upsert_dedup # TODO chris: SCD Type 1 can be implemented later
      - append_dedup # SCD Type 1 & 2
  OAuth2Specification:
    type: object
    additionalProperties: true
    description: An object containing any metadata needed to describe this connector's Oauth flow. Deprecated, switching to advanced_auth instead
    properties:
      rootObject:
        description:
          "A list of strings representing a pointer to the root object which contains any oauth parameters in the ConnectorSpecification.

          Examples:

          if oauth parameters were contained inside the top level, rootObject=[]
          If they were nested inside another object {'credentials': {'app_id' etc...}, rootObject=['credentials']
          If they were inside a oneOf {'switch': {oneOf: [{client_id...}, {non_oauth_param]}},  rootObject=['switch', 0]
          "
        type: array
        items:
          oneOf:
            - type: string
            - type: integer

      oauthFlowInitParameters:
        description:
          "Pointers to the fields in the rootObject needed to obtain the initial refresh/access tokens for the OAuth flow.
          Each inner array represents the path in the rootObject of the referenced field.
          For example.
          Assume the rootObject contains params 'app_secret', 'app_id' which are needed to get the initial refresh token.
          If they are not nested in the rootObject, then the array would look like this [['app_secret'], ['app_id']]
          If they are nested inside an object called 'auth_params' then this array would be [['auth_params', 'app_secret'], ['auth_params', 'app_id']]"
        type: array
        items:
          description: A list of strings denoting a pointer into the rootObject for where to find this property
          type: array
          items:
            type: string
      oauthFlowOutputParameters:
        description:
          "Pointers to the fields in the rootObject which can be populated from successfully completing the oauth flow using the init parameters.
          This is typically a refresh/access token.
          Each inner array represents the path in the rootObject of the referenced field."
        type: array
        items:
          description: A list of strings denoting a pointer into the rootObject for where to find this property
          type: array
          items:
            type: string
  ConnectorSpecification:
    type: object
    additionalProperties: true
    description: Specification of a connector (source/destination)
    required:
      - connectionSpecification
    properties:
      # todo - split ConnectorSpecification into general vs connector type sections of properties
      # General Properties (Common to all connectors)
      # todo (cgardens) - prefer snake case for field names.
      documentationUrl:
        type: string
        format: uri
      # todo (cgardens) - prefer snake case for field names.
      changelogUrl:
        type: string
        format: uri
      # todo (cgardens) - prefer snake case for field names.
      connectionSpecification:
        description: ConnectorDefinition specific blob. Must be a valid JSON string.
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      # Connector Type Properties (Common to all connectors from same type)
      # Source Connectors Properties
      supportsIncremental:
        description: (deprecated) If the connector supports incremental mode or not.
        type: boolean
      # Destination Connectors Properties
      # Normalization is currently implemented using dbt, so it requires `supportsDBT` to be true for this to be true.
      supportsNormalization:
        description: If the connector supports normalization or not.
        type: boolean
        default: false
      supportsDBT:
        description: If the connector supports DBT or not.
        type: boolean
        default: false
      supported_destination_sync_modes:
        description: List of destination sync modes supported by the connector
        type: array
        items:
          "$ref": "#/definitions/DestinationSyncMode"
      authSpecification:
        description: deprecated, switching to advanced_auth instead
        type: object
        properties:
          auth_type:
            type: string
            enum: ["oauth2.0"] # Future auth types should be added here
          oauth2Specification:
            description: If the connector supports OAuth, this field should be non-null.
            "$ref": "#/definitions/OAuth2Specification"
      advanced_auth:
        description: |-
          Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.
            - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode.
            - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the
            server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical
            inputs anymore and the auth process is faster and easier to complete.
        type: object
        properties:
          auth_flow_type:
            type: string
            enum: ["oauth2.0", "oauth1.0"] # Future auth types should be added here
          predicate_key:
            description: Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.
            type: array
            items:
              type: string
          predicate_value:
            description: Value of the predicate_key fields for the advanced auth to be applicable.
            type: string
          oauth_config_specification:
            "$ref": "#/definitions/OAuthConfigSpecification"
      protocol_version:
        description: "the Airbyte Protocol version supported by the connector. Protocol versioning uses SemVer. "
        type: string
  OAuthConfigSpecification:
    type: object
    additionalProperties: true
    properties:
      oauth_user_input_from_connector_config_specification:
        description: |-
          OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.
          Must be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification
          using special annotation 'path_in_connector_config'.
          These are input values the user is entering through the UI to authenticate to the connector, that might also shared
          as inputs for syncing data via the connector.

          Examples:

          if no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]
          if connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,
            oauth_user_input_from_connector_config_specification={
              app_id: {
                type: string
                path_in_connector_config: ['app_id']
              }
            }
          if connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,
            oauth_user_input_from_connector_config_specification={
              app_id: {
                type: string
                path_in_connector_config: ['info', 'app_id']
              }
            }
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      complete_oauth_output_specification:
        description: |-
          OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are
          returned by the distant OAuth APIs.
          Must be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.
          For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,

          Examples:

              complete_oauth_output_specification={
                refresh_token: {
                  type: string,
                  path_in_connector_config: ['credentials', 'refresh_token']
                }
              }
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      complete_oauth_server_input_specification:
        description: |-
          OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.
          Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the
          server when completing an OAuth flow (typically exchanging an auth code for refresh token).

          Examples:

              complete_oauth_server_input_specification={
                client_id: {
                  type: string
                },
                client_secret: {
                  type: string
                }
              }
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      complete_oauth_server_output_specification:
        description: |-
          OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that
          also need to be merged back into the connector configuration at runtime.
          This is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that
          are necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore
          they would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)
          Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the
          connector when using OAuth flow APIs.
          These fields are to be merged back to `ConnectorSpecification.connectionSpecification`.
          For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,

          Examples:

                complete_oauth_server_output_specification={
                  client_id: {
                    type: string,
                    path_in_connector_config: ['credentials', 'client_id']
                  },
                  client_secret: {
                    type: string,
                    path_in_connector_config: ['credentials', 'client_secret']
                  }
                }
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode