Merge remote-tracking branch 'origin/master' into flash1293/refactor-…

…connectorview

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.28
+current_version = 0.40.29
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.28
+VERSION=0.40.29
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/workflows/gradle.yml

@@ -124,6 +124,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Set up CI Gradle Properties
         run: |
@@ -229,6 +230,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Install Pyenv
         run: python3 -m pip install virtualenv==16.7.9 --user
@@ -412,6 +414,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Set up CI Gradle Properties
         run: |
@@ -535,6 +538,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Delete default old docker and replace it with a new one
         shell: bash
@@ -666,6 +670,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - uses: actions/setup-node@v3
         with:
@@ -800,6 +805,7 @@ jobs:
         if: always()
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Publish Platform Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -945,6 +951,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Install unzip
         shell: bash
@@ -1021,6 +1028,7 @@ jobs:
         if: always()
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Publish Kube Test Results
         id: kube-results
@@ -1191,6 +1199,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - uses: actions/setup-java@v1
         with:

.github/workflows/run-qa-engine.yml

@@ -0,0 +1,30 @@
+name: Run QA Engine
+
+on:
+  workflow_dispatch:
+  schedule:
+    # 1pm UTC is 6am PDT.
+    # same time as Generate Build Report
+    - cron: "0 13 * * *"
+
+jobs:
+  run-qa-engine:
+    name: "Run QA Engine"
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Setup Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.PROD_SPEC_CACHE_SA_KEY }}
+          export_default_credentials: true
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Install ci-connector-ops package
+        run: pip install --quiet -e ./tools/ci_connector_ops
+      - name: Run QA Engine
+        run: run-qa-engine

.github/workflows/test-command.yml

@@ -95,19 +95,12 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.9"
-      - name: Install Pyenv
-        run: |
-          python3 -m pip install --quiet virtualenv==16.7.9 --user
-          python3 -m virtualenv venv
-          source venv/bin/activate
       - name: Install CI scripts
         # all CI python packages have the prefix "ci_"
         run: |
-          source venv/bin/activate
           pip install --quiet -e ./tools/ci_*
       - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
         run: |
-          source venv/bin/activate
           ci_credentials ${{ github.event.inputs.connector }} write-to-storage
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
@@ -117,7 +110,6 @@ jobs:
           fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
-
       - name: Test ${{ github.event.inputs.connector }}
         id: test
         env:
@@ -132,7 +124,6 @@ jobs:
       - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}
         if: always()
         run: |
-          source venv/bin/activate
           ci_credentials ${{ github.event.inputs.connector }} update-secrets
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then

airbyte-api/src/main/openapi/config.yaml

@@ -754,6 +754,27 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+
+  /v1/sources/write_discover_catalog_result:
+    post:
+      tags:
+        - source
+        - internal
+      summary: Should only called from worker, to write result from discover activity back to DB.
+      operationId: writeDiscoverCatalogResult
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SourceDiscoverSchemaWriteRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful Operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DiscoverCatalogResult"
   /v1/destination_definitions/update:
     post:
       tags:
@@ -1989,6 +2010,28 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/jobs/get_last_replication_job:
+    post:
+      tags:
+        - jobs
+      operationId: getLastReplicationJob
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ConnectionIdRequestBody"
+        required: true
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JobOptionalRead"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "422":
+          $ref: "#/components/responses/InvalidInputResponse"
   /v1/jobs/get_light:
     post:
       tags:
@@ -2790,6 +2833,21 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/SourceRead"
+    SourceDiscoverSchemaWriteRequestBody:
+      description: to write this requested object to database.
+      type: object
+      required:
+        - catalog
+      properties:
+        catalog:
+          $ref: "#/components/schemas/AirbyteCatalog"
+        sourceId:
+          $ref: "#/components/schemas/SourceId"
+        connectorVersion:
+          type: string
+        configurationHash:
+          type: string
+
     SourceDiscoverSchemaRead:
       description: Returns the results of a discover catalog job. If the job was not successful, the catalog field will not be present. jobInfo will aways be present and its status be used to determine if the job was successful or not.
       type: object
@@ -3907,6 +3965,9 @@ components:
         updatedAt:
           type: integer
           format: int64
+        startedAt:
+          type: integer
+          format: int64
         status:
           $ref: "#/components/schemas/JobStatus"
         resetConfig:
@@ -4142,6 +4203,11 @@ components:
       properties:
         job:
           $ref: "#/components/schemas/JobRead"
+    JobOptionalRead:
+      type: object
+      properties:
+        job:
+          $ref: "#/components/schemas/JobRead"
     JobDebugInfoRead:
       type: object
       required:
@@ -4963,6 +5029,14 @@ components:
       properties:
         succeeded:
           type: boolean
+    DiscoverCatalogResult:
+      type: object
+      required:
+        - catalogId
+      properties:
+        catalogId:
+          type: string
+          format: uuid
     AttemptNormalizationStatusReadList:
       type: object
       properties:

airbyte-bootloader/Dockerfile

@@ -1,7 +1,7 @@
 ARG JDK_IMAGE=airbyte/airbyte-base-java-image:1.0
 FROM ${JDK_IMAGE}
 
-ARG VERSION=0.40.28
+ARG VERSION=0.40.29
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

airbyte-cdk/python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.21.0
+current_version = 0.22.0
 commit = False
 
 [bumpversion:file:setup.py]

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.22.0
+Surface the resolved manifest in the CDK
+
 ## 0.21.0
 Add AvailabilityStrategy concept and use check_availability within CheckStream
 

airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -81,6 +81,10 @@ def __init__(self, source_config: ConnectionDefinition, debug: bool = False, con
         if unknown_fields:
             raise InvalidConnectorDefinitionException(f"Found unknown top-level fields: {unknown_fields}")
 
+    @property
+    def resolved_manifest(self) -> Mapping[str, Any]:
+        return self._new_source_config
+
     @property
     def connection_checker(self) -> ConnectionChecker:
         check = self._new_source_config["check"] if self.construct_using_pydantic_models else self._legacy_source_config["check"]

airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -359,7 +359,10 @@ def create_declarative_stream(model: DeclarativeStreamModel, config: Config, **k
     if model.schema_loader:
         schema_loader = _create_component_from_model(model=model.schema_loader, config=config)
     else:
-        schema_loader = DefaultSchemaLoader(config=config, options=model.options)
+        options = model.options or {}
+        if "name" not in options:
+            options["name"] = model.name
+        schema_loader = DefaultSchemaLoader(config=config, options=options)
 
     transformations = []
     if model.transformations:

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.21.0",
+    version="0.22.0",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py

@@ -1011,3 +1011,37 @@ def test_add_fields(self):
             )
         ]
         assert stream.transformations == expected
+
+    def test_default_schema_loader(self):
+        component_definition = {
+            "type": "DeclarativeStream",
+            "name": "test",
+            "primary_key": [],
+            "retriever": {
+                "type": "SimpleRetriever",
+                "name": "test",
+                "primary_key": [],
+                "requester": {
+                    "type": "HttpRequester",
+                    "name": "test",
+                    "url_base": "http://localhost:6767/",
+                    "path": "items/",
+                    "request_options_provider": {
+                        "request_parameters": {},
+                        "request_headers": {},
+                        "request_body_json": {},
+                        "type": "InterpolatedRequestOptionsProvider",
+                    },
+                    "authenticator": {"type": "BearerAuthenticator", "api_token": "{{ config['api_key'] }}"},
+                },
+                "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_pointer": ["items"]}},
+                "paginator": {"type": "NoPagination"},
+            },
+        }
+        resolved_manifest = resolver.preprocess_manifest(component_definition)
+        propagated_source_config = ManifestComponentTransformer().propagate_types_and_options("", resolved_manifest, {})
+        stream = factory.create_component(
+            model_type=DeclarativeStreamModel, component_definition=propagated_source_config, config=input_config
+        )
+        schema_loader = stream.schema_loader
+        assert schema_loader.default_loader._get_json_filepath().split("/")[-1] == f"{stream.name}.json"

airbyte-commons-worker/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java

@@ -11,6 +11,7 @@
 import io.airbyte.api.client.AirbyteApiClient;
 import io.airbyte.api.client.generated.ConnectionApi;
 import io.airbyte.api.client.generated.DestinationApi;
+import io.airbyte.api.client.generated.JobsApi;
 import io.airbyte.api.client.generated.SourceApi;
 import io.airbyte.api.client.generated.WorkspaceApi;
 import io.airbyte.api.client.invoker.generated.ApiClient;
@@ -72,6 +73,11 @@ public SourceApi sourceApi(@Named("apiClient") final ApiClient apiClient) {
     return new SourceApi(apiClient);
   }
 
+  @Singleton
+  public JobsApi jobsApi(@Named("apiClient") final ApiClient apiClient) {
+    return new JobsApi(apiClient);
+  }
+
   @Singleton
   public DestinationApi destinationApi(final ApiClient apiClient) {
     return new DestinationApi(apiClient);