airbyte_protocol.yaml

---
"$schema": http://json-schema.org/draft-07/schema#
"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml
title: AirbyteProtocol
type: object
description: AirbyteProtocol structs
properties:
  airbyte_message:
    "$ref": "#/definitions/AirbyteMessage"
  configured_airbyte_catalog:
    "$ref": "#/definitions/ConfiguredAirbyteCatalog"
definitions:
  AirbyteMessage:
    type: object
    additionalProperties: true
    required:
      - type
    properties:
      type:
        description: "Message type"
        type: string
        enum:
          - RECORD
          - STATE
          - LOG
          - SPEC
          - CONNECTION_STATUS
          - CATALOG
      log:
        description: "log message: any kind of logging you want the platform to know about."
        "$ref": "#/definitions/AirbyteLogMessage"
      spec:
        "$ref": "#/definitions/ConnectorSpecification"
      # todo (cgardens) - prefer snake case for field names.
      connectionStatus:
        "$ref": "#/definitions/AirbyteConnectionStatus"
      catalog:
        description: "catalog message: the catalog"
        "$ref": "#/definitions/AirbyteCatalog"
      record:
        description: "record message: the record"
        "$ref": "#/definitions/AirbyteRecordMessage"
      state:
        description: "schema message: the state. Must be the last message produced. The platform uses this information"
        "$ref": "#/definitions/AirbyteStateMessage"
  AirbyteRecordMessage:
    type: object
    additionalProperties: true
    required:
      - stream
      - data
      - emitted_at
    properties:
      stream:
        description: "the name of this record's stream"
        type: string
      data:
        description: "the record data"
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      emitted_at:
        description: "when the data was emitted from the source. epoch in millisecond."
        type: integer
      namespace:
        description: "the namespace of this record's stream"
        type: string
  AirbyteStateMessage:
    type: object
    additionalProperties: true
    required:
      - data
    properties:
      data:
        description: "the state data"
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
  AirbyteLogMessage:
    type: object
    additionalProperties: true
    required:
      - level
      - message
    properties:
      level:
        description: "the type of logging"
        type: string
        enum:
          - FATAL
          - ERROR
          - WARN
          - INFO
          - DEBUG
          - TRACE
      message:
        description: "the log message"
        type: string
  AirbyteConnectionStatus:
    description: Airbyte connection status
    type: object
    additionalProperties: true
    required:
      - status
    properties:
      status:
        type: string
        enum:
          - SUCCEEDED
          - FAILED
      message:
        type: string
  AirbyteCatalog:
    description: Airbyte stream schema catalog
    type: object
    additionalProperties: true
    required:
      - streams
    properties:
      streams:
        type: array
        items:
          "$ref": "#/definitions/AirbyteStream"
  AirbyteStream:
    type: object
    additionalProperties: true
    required:
      - name
      - json_schema
      # todo (cgardens) - make required once sources are migrated
      # - supported_sync_modes
    properties:
      name:
        type: string
        description: Stream's name.
      json_schema:
        description: Stream schema using Json Schema specs.
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      supported_sync_modes:
        type: array
        items:
          "$ref": "#/definitions/SyncMode"
      source_defined_cursor:
        description: If the source defines the cursor field, then any other cursor field inputs will be ignored. If it does not, either the user_provided one is used, or the default one is used as a backup.
        type: boolean
      default_cursor_field:
        description: Path to the field that will be used to determine if a record is new or modified since the last sync. If not provided by the source, the end user will have to specify the comparable themselves.
        type: array
        items:
          type: string
      source_defined_primary_key:
        description: If the source defines the primary key, paths to the fields that will be used as a primary key. If not provided by the source, the end user will have to specify the primary key themselves.
        type: array
        items:
          type: array
          items:
            type: string
      namespace:
        description: Optional Source-defined namespace. Currently only used by JDBC destinations to determine what schema to write to. Airbyte streams from the same sources should have the same namespace.
        type: string
  ConfiguredAirbyteCatalog:
    description: Airbyte stream schema catalog
    type: object
    additionalProperties: true
    required:
      - streams
    properties:
      streams:
        type: array
        items:
          "$ref": "#/definitions/ConfiguredAirbyteStream"
  ConfiguredAirbyteStream:
    type: object
    additionalProperties: true
    required:
      - stream
      - sync_mode
      - destination_sync_mode
    properties:
      stream:
        "$ref": "#/definitions/AirbyteStream"
      sync_mode:
        "$ref": "#/definitions/SyncMode"
        default: full_refresh
      cursor_field:
        description: Path to the field that will be used to determine if a record is new or modified since the last sync. This field is REQUIRED if `sync_mode` is `incremental`. Otherwise it is ignored.
        type: array
        items:
          type: string
      destination_sync_mode:
        "$ref": "#/definitions/DestinationSyncMode"
        default: append
      primary_key:
        description: Paths to the fields that will be used as primary key. This field is REQUIRED if `destination_sync_mode` is `*_dedup`. Otherwise it is ignored.
        type: array
        items:
          type: array
          items:
            type: string
  SyncMode:
    type: string
    enum:
      - full_refresh
      - incremental
  DestinationSyncMode:
    type: string
    enum:
      - append
      - overwrite
      #- upsert_dedup # TODO chris: SCD Type 1 can be implemented later
      - append_dedup # SCD Type 1 & 2
  OAuth2Specification:
    description: An object containing any metadata needed to describe this connector's Oauth flow. Deprecated, switching to advanced_auth instead
    type: object
    additionalProperties: true
    properties:
      rootObject:
        description:
          "A list of strings representing a pointer to the root object which contains any oauth parameters in the ConnectorSpecification.

          Examples:

          if oauth parameters were contained inside the top level, rootObject=[]
          If they were nested inside another object {'credentials': {'app_id' etc...}, rootObject=['credentials']
          If they were inside a oneOf {'switch': {oneOf: [{client_id...}, {non_oauth_param]}},  rootObject=['switch', 0]
          "
        type: array
        items:
          oneOf:
            - type: string
            - type: integer

      oauthFlowInitParameters:
        description:
          "Pointers to the fields in the rootObject needed to obtain the initial refresh/access tokens for the OAuth flow.
          Each inner array represents the path in the rootObject of the referenced field.
          For example.
          Assume the rootObject contains params 'app_secret', 'app_id' which are needed to get the initial refresh token.
          If they are not nested in the rootObject, then the array would look like this [['app_secret'], ['app_id']]
          If they are nested inside an object called 'auth_params' then this array would be [['auth_params', 'app_secret'], ['auth_params', 'app_id']]"
        type: array
        items:
          description: A list of strings denoting a pointer into the rootObject for where to find this property
          type: array
          items:
            type: string
      oauthFlowOutputParameters:
        description:
          "Pointers to the fields in the rootObject which can be populated from successfully completing the oauth flow using the init parameters.
          This is typically a refresh/access token.
          Each inner array represents the path in the rootObject of the referenced field."
        type: array
        items:
          description: A list of strings denoting a pointer into the rootObject for where to find this property
          type: array
          items:
            type: string
  ConnectorSpecification:
    description: Specification of a connector (source/destination)
    type: object
    required:
      - connectionSpecification
    additionalProperties: true
    properties:
      # todo - split ConnectorSpecification into general vs connector type sections of properties
      # General Properties (Common to all connectors)
      # todo (cgardens) - prefer snake case for field names.
      documentationUrl:
        type: string
        format: uri
      # todo (cgardens) - prefer snake case for field names.
      changelogUrl:
        type: string
        format: uri
      # todo (cgardens) - prefer snake case for field names.
      connectionSpecification:
        description: ConnectorDefinition specific blob. Must be a valid JSON string.
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      # Connector Type Properties (Common to all connectors from same type)
      # Source Connectors Properties
      supportsIncremental:
        description: If the connector supports incremental mode or not.
        type: boolean
      # Destination Connectors Properties
      # Normalization is currently implemented using dbt so it requires `supportsDBT` to be true for this to be true.
      supportsNormalization:
        description: If the connector supports normalization or not.
        type: boolean
        default: false
      supportsDBT:
        description: If the connector supports DBT or not.
        type: boolean
        default: false
      supported_destination_sync_modes:
        description: List of destination sync modes supported by the connector
        type: array
        items:
          "$ref": "#/definitions/DestinationSyncMode"
      authSpecification:
        description: deprecated, switching to advanced_auth instead
        type: object
        properties:
          auth_type:
            type: string
            enum: ["oauth2.0"] # Future auth types should be added here
          oauth2Specification:
            description: If the connector supports OAuth, this field should be non-null.
            "$ref": "#/definitions/OAuth2Specification"
      advanced_auth:
        description: |-
          Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.
            - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode.
            - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the
            server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical
            inputs anymore and the auth process is faster and easier to complete.
        type: object
        properties:
          auth_flow_type:
            type: string
            enum: ["oauth2.0", "oauth1.0"] # Future auth types should be added here
          predicate_key:
            description: Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.
            type: array
            items:
              type: string
          predicate_value:
            description: Value of the predicate_key fields for the advanced auth to be applicable.
            type: string
          oauth_config_specification:
            "$ref": "#/definitions/OAuthConfigSpecification"
  OAuthConfigSpecification:
    type: object
    properties:
      oauth_user_input_from_connector_config_specification:
        description: |-
          OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.
          Must be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification
          using special annotation 'path_in_connector_config'.
          These are input values the user is entering through the UI to authenticate to the connector, that might also shared
          as inputs for syncing data via the connector.

          Examples:

          if no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]
          if connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,
            oauth_user_input_from_connector_config_specification={
              app_id: {
                type: string
                path_in_connector_config: ['app_id']
              }
            }
          if connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,
            oauth_user_input_from_connector_config_specification={
              app_id: {
                type: string
                path_in_connector_config: ['info', 'app_id']
              }
            }
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      complete_oauth_output_specification:
        description: |-
          OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are
          returned by the distant OAuth APIs.
          Must be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.
          For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,

          Examples:

              complete_oauth_output_specification={
                refresh_token: {
                  type: string,
                  path_in_connector_config: ['credentials', 'refresh_token']
                }
              }
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      complete_oauth_server_input_specification:
        description: |-
          OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.
          Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the
          server when completing an OAuth flow (typically exchanging an auth code for refresh token).

          Examples:

              complete_oauth_server_input_specification={
                client_id: {
                  type: string
                },
                client_secret: {
                  type: string
                }
              }
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      complete_oauth_server_output_specification:
        description: |-
          OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that
          also need to be merged back into the connector configuration at runtime.
          This is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that
          are necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore
          they would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)
          Must be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the
          connector when using OAuth flow APIs.
          These fields are to be merged back to `ConnectorSpecification.connectionSpecification`.
          For each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,

          Examples:

                complete_oauth_server_output_specification={
                  client_id: {
                    type: string,
                    path_in_connector_config: ['credentials', 'client_id']
                  },
                  client_secret: {
                    type: string,
                    path_in_connector_config: ['credentials', 'client_secret']
                  }
                }
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode