Merge branch 'master' into ddavydov/#20703-source-salesforce-include-…

…pk-in-properties-chunks

airbyte-cdk/python/.bumpversion.cfg

@@ -3,3 +3,4 @@ current_version = 0.29.0
 commit = False
 
 [bumpversion:file:setup.py]
+[bumpversion:file:Dockerfile]

airbyte-cdk/python/Dockerfile

@@ -0,0 +1,36 @@
+FROM python:3.9.11-alpine3.15 as base
+
+# build and load all requirements
+FROM base as builder
+WORKDIR /airbyte/integration_code
+
+# upgrade pip to the latest version
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
+
+# install airbyte-cdk
+RUN pip install --prefix=/install airbyte-cdk==0.29.0
+
+# build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code
+
+# copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# copy payload code only
+COPY source_declarative_manifest/main.py ./
+
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
+
+# needs to be the same as CDK
+LABEL io.airbyte.version=0.29.0
+LABEL io.airbyte.name=airbyte/source-declarative-manifest

airbyte-cdk/python/source_declarative_manifest/README.md

@@ -0,0 +1,49 @@
+# Declarative manifest source
+
+This is a generic source that takes the declarative manifest via a key `__injected_declarative_manifest` of its config.
+
+## Execution
+This entrypoint is used for connectors created by the connector builder. These connector's spec is defined in their manifest, which is defined in the config's "__injected_declarative_manifest" field. This allows this entrypoint to be used with any connector manifest.
+
+The spec operation is not supported because the config is not known when running a spec.
+
+## Local development
+
+#### Building
+
+You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow.
+
+To build using Gradle, from the Airbyte repository root, run:
+
+```
+./gradlew airbyte-cdk:python:build
+```
+
+### Locally running the connector
+
+```
+python main.py check --config secrets/config.json
+python main.py discover --config secrets/config.json
+python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json
+```
+
+### Locally running the connector docker image
+
+#### Build
+
+First, make sure you build the latest Docker image:
+```
+./gradlew airbyte-cdk:python:airbyteDocker
+```
+
+The docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in the Dockerfile.
+
+#### Run
+
+Then run any of the connector commands as follows:
+
+```
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-declarative-manifest:dev check --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-declarative-manifest:dev discover --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-declarative-manifest:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
+```

airbyte-cdk/python/source_declarative_manifest/main.py

@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import sys
+from typing import List
+
+from airbyte_cdk.connector import BaseConnector
+from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
+from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+
+
+def create_manifest(args: List[str]):
+    parsed_args = AirbyteEntrypoint.parse_args(args)
+    if parsed_args.command == "spec":
+        raise ValueError("spec command is not supported for injected declarative manifest")
+
+    config = BaseConnector.read_config(parsed_args.config)
+    if "__injected_declarative_manifest" not in config:
+        raise ValueError(
+            f"Invalid config: `__injected_declarative_manifest` should be provided at the root of the config but config only has keys {list(config.keys())}"
+        )
+    return ManifestDeclarativeSource(config.get("__injected_declarative_manifest"))
+
+
+if __name__ == "__main__":
+    source = create_manifest(sys.argv[1:])
+    launch(source, sys.argv[1:])

airbyte-cdk/python/unit_tests/test_source_declarative_manifest.py

@@ -0,0 +1,98 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import copy
+import json
+
+import pytest
+from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
+from source_declarative_manifest.main import create_manifest
+
+CONFIG = {
+    "__injected_declarative_manifest": {
+        "version": "0.1.0",
+        "definitions": {
+            "selector": {
+                "extractor": {
+                    "field_path": []
+                }
+            },
+            "requester": {
+                "url_base": "https://test.com/api",
+                "http_method": "GET"
+            },
+            "retriever": {
+                "record_selector": {
+                    "$ref": "#/definitions/selector"
+                },
+                "requester": {
+                    "$ref": "#/definitions/requester"
+                }
+            },
+            "base_stream": {
+                "retriever": {
+                    "$ref": "#/definitions/retriever"
+                }
+            },
+            "data_stream": {
+                "$ref": "#/definitions/base_stream",
+                "$parameters": {
+                    "name": "data",
+                    "path": "/data"
+                }
+            },
+        },
+        "streams": [
+            "#/definitions/data_stream",
+        ],
+        "check": {
+            "stream_names": [
+                "data",
+            ]
+        },
+        "spec": {
+            "type": "Spec",
+            "documentation_url": "https://test.com/doc",
+            "connection_specification": {
+                "$schema": "http://json-schema.org/draft-07/schema#",
+                "title": "Test Spec",
+                "type": "object",
+                "additionalProperties": True,
+                "properties": {}
+            }
+        }
+    }
+}
+
+
+@pytest.fixture
+def valid_config_file(tmp_path):
+    config_file = tmp_path / "config.json"
+    config_file.write_text(json.dumps(CONFIG))
+    return config_file
+
+
+@pytest.fixture
+def config_file_without_injection(tmp_path):
+    config = copy.deepcopy(CONFIG)
+    del config["__injected_declarative_manifest"]
+
+    config_file = tmp_path / "config.json"
+    config_file.write_text(json.dumps(config))
+    return config_file
+
+
+def test_on_spec_command_then_raise_value_error():
+    with pytest.raises(ValueError):
+        create_manifest(["spec"])
+
+
+def test_given_no_injected_declarative_manifest_then_raise_value_error(config_file_without_injection):
+    with pytest.raises(ValueError):
+        create_manifest(["check", "--config", str(config_file_without_injection)])
+
+
+def test_given_injected_declarative_manifest_then_return_declarative_manifest(valid_config_file):
+    source = create_manifest(["check", "--config", str(valid_config_file)])
+    assert isinstance(source, ManifestDeclarativeSource)

airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/manifest.yaml.hbs

@@ -5,7 +5,7 @@ definitions:
     type: RecordSelector
     extractor:
       type: DpathExtractor
-      field_pointer: []
+      field_path: []
   requester:
     type: HttpRequester
     url_base: "https://example.com"

docs/deploying-airbyte/on-kubernetes.md

@@ -1,7 +1,12 @@
-# Deploy Airbyte on Kubernetes
+# (Deprecated) Deploy Airbyte on Kubernetes via Kustomize
+
+:::caution
+This deployment method uses Kustomize and is only supported up to [Airbyte version `0.40.32`](https://github.com/airbytehq/airbyte/releases/tag/v0.40.32). For existing deployments, check out [commit `21a7e102183e20d2d4998ea70c2a8fe4eac8921b`](https://github.com/airbytehq/airbyte/commit/21a7e102183e20d2d4998ea70c2a8fe4eac8921b) to continue deploying using Kustomize. For new deployments, [deploy Airbyte on Kubernetes via Helm](/deploying-airbyte/on-kubernetes-via-helm.md).
+:::
 
 This page guides you through deploying Airbyte Open Source on Kubernetes. 
 
+
 ## Requirements
 
 To test locally, you can use one of the following: