Merge branch 'master' into leti/experiment-add-source-in-signup

* master: (130 commits)
  ami-0d648081937c75a73 -> ami-06cf12549e3d9c522 (#18667)
  Rename maxCursor and add comment (#18570)
  Fix source spec diff (#18645)
  Source S3: use AirbyteTracedException (#18602)
  Add endpoint to retrieve manifest template (#18578)
  Source Mailjet SMS: publish PR and add source def (#18620)
  🎉 New Connector: Zendesk Sell [python cdk] (#17888)
  More comprehensive temporal error message (#18608)
  🎉 New Source: Mailjet SMS (#18345)
  Fix ConfiguredCatalog for Resets (#18625)
  Bmoric/extract healt api (#18523)
  Add versioning logging (#18618)
  🐛 Lowcode: ListStreamSlicer and SubstreamSlicer should get the stream_slice from the arguments (#18574)
  Improved the Oracle cloud deployment guide (#18615)
  Remove workflow version check (#18613)
  Protocol Change: `AirbyteControlMessage.ConnectorConfig` (#17907)
  Replace `recipesLink` link with `tutorialsLink` (#18616)
  🎉 New Source: Waiteraid [low-code cdk] (#18165)
  🎉 New Destination: Typesense (#18349)
  Clean up build.gradle. (#18555)
  ...

.github/workflows/deploy-docs-site.yml

@@ -1,22 +1,51 @@
-name: Deploy docs.airbyte.com [DUMMY workflow]
+name: Deploy docs.airbyte.com
 
 on:
-  ## XXX uncomment the following when this code gets in good shape
-  #push:
-  #  branches:
-  #    - master
-  #  paths:
-  #    - 'docs/**'
+  push:
+    branches:
+      - master
+    paths:
+      - 'docs/**'
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 jobs:
   dummy-job:
-    name: A placeholder job
+    name: Deploy Docs Assets
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
-      - name: A placeholder step
-        shell: bash
-        run: |-
-          echo "Hello from 'Deploy Docs' workflow!"
+      - name: Check out the repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # Node.js is needed for Yarn
+      - name: Setup Yarn
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16.14.0'
+          cache: 'yarn'
+          cache-dependency-path: docusaurus
+
+      - name: Run Docusaurus
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./tools/bin/deploy_docusaurus
+
+      - name: Notify Slack channel on failure
+        uses: abinoda/slack-action@master
+        if: failure()
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+        with:
+          # 'C03BEADRPNY' channel => '#oss-master-build-failure'
+          args: >-
+            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
+            {\"type\":\"divider\"},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"OSS Docs build fails on the latest master :bangbang: \n\n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
+            {\"type\":\"divider\"}]}
+

.github/workflows/publish-connector-command.yml

@@ -67,7 +67,7 @@ jobs:
 #          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
 #          github-token: ${{ needs.find_valid_pat.outputs.pat }}
 #          # 80 gb disk
-#          ec2-image-id: ami-0d648081937c75a73
+#          ec2-image-id: ami-06cf12549e3d9c522
 #  bump-build-test-connector:
 #    needs: start-bump-build-test-connector-runner
 #    runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}

.github/workflows/test-command.yml

@@ -62,7 +62,7 @@ jobs:
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ needs.find_valid_pat.outputs.pat }}
           # 80 gb disk
-          ec2-image-id: ami-0d648081937c75a73
+          ec2-image-id: ami-06cf12549e3d9c522
   integration-test:
     timeout-minutes: 240
     needs: start-test-runner

.github/workflows/test-performance-command.yml

@@ -65,7 +65,7 @@ jobs:
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ needs.find_valid_pat.outputs.pat }}
           # 80 gb disk
-          ec2-image-id: ami-0d648081937c75a73
+          ec2-image-id: ami-06cf12549e3d9c522
   performance-test:
     timeout-minutes: 240
     needs: start-test-runner

airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java

@@ -16,6 +16,7 @@
 import io.airbyte.api.client.generated.SourceApi;
 import io.airbyte.api.client.generated.SourceDefinitionApi;
 import io.airbyte.api.client.generated.SourceDefinitionSpecificationApi;
+import io.airbyte.api.client.generated.StateApi;
 import io.airbyte.api.client.generated.WorkspaceApi;
 import io.airbyte.api.client.invoker.generated.ApiClient;
 
@@ -46,8 +47,8 @@ public class AirbyteApiClient {
   private final WorkspaceApi workspaceApi;
   private final HealthApi healthApi;
   private final DbMigrationApi dbMigrationApi;
-
   private final AttemptApi attemptApi;
+  private final StateApi stateApi;
 
   public AirbyteApiClient(final ApiClient apiClient) {
     connectionApi = new ConnectionApi(apiClient);
@@ -64,6 +65,7 @@ public AirbyteApiClient(final ApiClient apiClient) {
     healthApi = new HealthApi(apiClient);
     dbMigrationApi = new DbMigrationApi(apiClient);
     attemptApi = new AttemptApi(apiClient);
+    stateApi = new StateApi(apiClient);
   }
 
   public ConnectionApi getConnectionApi() {
@@ -122,4 +124,8 @@ public AttemptApi getAttemptApi() {
     return attemptApi;
   }
 
+  public StateApi getStateApi() {
+    return stateApi;
+  }
+
 }

airbyte-api/src/main/openapi/config.yaml

@@ -63,6 +63,8 @@ tags:
     description: Export/Import Airbyte Configuration and Database resources.
   - name: attempt
     description: Interactions with attempt related resources.
+  - name: state
+    description: Interactions with state related resources.
 
 paths:
   /v1/workspaces/create:
@@ -1389,7 +1391,7 @@ paths:
   /v1/state/get:
     post:
       tags:
-        - connection
+        - state
       summary: Fetch the current state for a connection.
       operationId: getState
       requestBody:
@@ -1412,7 +1414,7 @@ paths:
   /v1/state/create_or_update:
     post:
       tags:
-        - connection
+        - state
         - internal
       summary: Create or update the state for a connection.
       operationId: createOrUpdateState
@@ -1994,7 +1996,7 @@ paths:
   /v1/web_backend/state/get_type:
     post:
       tags:
-        - connection
+        - web_backend
       summary: Fetch the current state type for a connection.
       operationId: getStateType
       requestBody:
@@ -2169,6 +2171,26 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/jobs/get_normalization_status:
+    post:
+      tags:
+        - jobs
+        - internal
+      summary: Get normalization status to determine if we can bypass normalization phase
+      operationId: getAttemptNormalizationStatusesForJob
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/JobIdRequestBody"
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/AttemptNormalizationStatusReadList"
+
   /v1/health:
     get:
       tags:
@@ -2242,6 +2264,7 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/InternalOperationResult"
+
 components:
   securitySchemes:
     bearerAuth:
@@ -4838,6 +4861,26 @@ components:
       properties:
         succeeded:
           type: boolean
+    AttemptNormalizationStatusReadList:
+      type: object
+      properties:
+        attemptNormalizationStatuses:
+          type: array
+          items:
+            $ref: "#/components/schemas/AttemptNormalizationStatusRead"
+    AttemptNormalizationStatusRead:
+      type: object
+      properties:
+        attemptNumber:
+          $ref: "#/components/schemas/AttemptNumber"
+        hasRecordsCommitted:
+          type: boolean
+        recordsCommitted:
+          type: integer
+          format: int64
+        hasNormalizationFailed:
+          type: boolean
+
     InvalidInputProperty:
       type: object
       required:

airbyte-bootloader/build.gradle

@@ -11,7 +11,7 @@ dependencies {
     implementation project(':airbyte-protocol:protocol-models')
     implementation project(':airbyte-persistence:job-persistence')
 
-    implementation 'io.temporal:temporal-sdk:1.8.1'
+    implementation libs.temporal.sdk
     implementation libs.flyway.core
 
     testImplementation libs.platform.testcontainers.postgresql

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.4.2
+Low-code: Fix off by one error with the stream slicers
+
+## 0.4.1
+Low-code: Fix a few bugs with the stream slicers
+
+## 0.4.0
+Low-code: Add support for custom error messages on error response filters
+
+## 0.3.0
+Publish python typehints via `py.typed` file. 
+
 ## 0.2.3
 - Propagate options to InterpolatedRequestInputProvider
 

airbyte-cdk/python/airbyte_cdk/models/__init__.py

@@ -1,2 +1,3 @@
 # generated by generate-protocol-files
 from .airbyte_protocol import *
+from .well_known_types import *

airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py

@@ -21,6 +21,7 @@ class Type(Enum):
     CONNECTION_STATUS = "CONNECTION_STATUS"
     CATALOG = "CATALOG"
     TRACE = "TRACE"
+    CONTROL = "CONTROL"
 
 
 class AirbyteRecordMessage(BaseModel):
@@ -97,6 +98,17 @@ class Config:
     failure_type: Optional[FailureType] = Field(None, description="The type of error")
 
 
+class OrchestratorType(Enum):
+    CONNECTOR_CONFIG = "CONNECTOR_CONFIG"
+
+
+class AirbyteControlConnectorConfigMessage(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    config: Dict[str, Any] = Field(..., description="the config items from this connector's spec to update")
+
+
 class Status(Enum):
     SUCCEEDED = "SUCCEEDED"
     FAILED = "FAILED"
@@ -203,6 +215,18 @@ class Config:
     error: Optional[AirbyteErrorTraceMessage] = Field(None, description="error trace message: the error object")
 
 
+class AirbyteControlMessage(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: OrchestratorType = Field(..., description="the type of orchestrator message", title="orchestrator type")
+    emitted_at: float = Field(..., description="the time in ms that the message was emitted")
+    connectorConfig: Optional[AirbyteControlConnectorConfigMessage] = Field(
+        None,
+        description="connector config orchestrator message: the updated config for the platform to store for this connector",
+    )
+
+
 class AirbyteStream(BaseModel):
     class Config:
         extra = Extra.allow
@@ -333,6 +357,10 @@ class Config:
         None,
         description="trace message: a message to communicate information about the status and performance of a connector",
     )
+    control: Optional[AirbyteControlMessage] = Field(
+        None,
+        description="connector config message: a message to communicate an updated configuration from a connector that should be persisted",
+    )
 
 
 class AirbyteProtocol(BaseModel):

airbyte-cdk/python/airbyte_cdk/models/well_known_types.py

@@ -0,0 +1,86 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+# generated by datamodel-codegen:
+#   filename:  well_known_types.yaml
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Union
+
+from pydantic import BaseModel, Field, constr
+
+
+class Model(BaseModel):
+    __root__: Any
+
+
+class String(BaseModel):
+    __root__: str = Field(..., description="Arbitrary text")
+
+
+class BinaryData(BaseModel):
+    __root__: constr(regex=r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$") = Field(
+        ...,
+        description="Arbitrary binary data. Represented as base64-encoded strings in the JSON transport. In the future, if we support other transports, may be encoded differently.\n",
+    )
+
+
+class Date(BaseModel):
+    __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}( BC)?$") = Field(
+        ..., description="RFC 3339§5.6's full-date format, extended with BC era support"
+    )
+
+
+class TimestampWithTimezone(BaseModel):
+    __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+\-]\d{1,2}:\d{2})( BC)?$") = Field(
+        ...,
+        description='An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339§5.6\'s date-time format, requiring a "T" separator, and extended with BC era support. Note that we do _not_ accept Unix epochs here.\n',
+    )
+
+
+class TimestampWithoutTimezone(BaseModel):
+    __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?( BC)?$") = Field(
+        ...,
+        description='Also known as a localdatetime, or just datetime. Under RFC 3339§5.6, this would be represented as `full-date "T" partial-time`, extended with BC era support.\n',
+    )
+
+
+class TimeWithTimezone(BaseModel):
+    __root__: constr(regex=r"^\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+\-]\d{1,2}:\d{2})$") = Field(..., description="An RFC 3339§5.6 full-time")
+
+
+class TimeWithoutTimezone(BaseModel):
+    __root__: constr(regex=r"^\d{2}:\d{2}:\d{2}(\.\d+)?$") = Field(..., description="An RFC 3339§5.6 partial-time")
+
+
+class NumberEnum(Enum):
+    Infinity = "Infinity"
+    _Infinity = "-Infinity"
+    NaN = "NaN"
+
+
+class Number(BaseModel):
+    __root__: Union[Any, NumberEnum] = Field(
+        ...,
+        description="Note the mix of regex validation for normal numbers, and enum validation for special values.",
+    )
+
+
+class IntegerEnum(Enum):
+    Infinity = "Infinity"
+    _Infinity = "-Infinity"
+    NaN = "NaN"
+
+
+class Integer(BaseModel):
+    __root__: Union[Any, IntegerEnum]
+
+
+class Boolean(BaseModel):
+    __root__: bool = Field(
+        ...,
+        description="Note the direct usage of a primitive boolean rather than string. Unlike Numbers and Integers, we don't expect unusual values  here.",
+    )