From 3c8bb42b1891703ca1f05c2add7bac4a3f631ba8 Mon Sep 17 00:00:00 2001 From: Peter Hu Date: Mon, 12 Dec 2022 10:27:43 -0800 Subject: [PATCH 001/134] Workflow for Approval Bot Dispatch (#20376) --- .../workflows/approve-and-merge-dispatch.yml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/approve-and-merge-dispatch.yml diff --git a/.github/workflows/approve-and-merge-dispatch.yml b/.github/workflows/approve-and-merge-dispatch.yml new file mode 100644 index 000000000000..8c7e212a3be8 --- /dev/null +++ b/.github/workflows/approve-and-merge-dispatch.yml @@ -0,0 +1,36 @@ +name: Approve and Merge Command Dispatch +on: + issue_comment: + types: [created] +jobs: + approveAndMergeDispatch: + runs-on: ubuntu-latest + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OCTAVIA_GITHUB_RUNNER_TOKEN }} \ + ${{ secrets.SUPERTOPHER_PAT }} + + - name: Auto Approve Slash Command Dispatch + uses: peter-evans/slash-command-dispatch@v3 + id: scd + with: + token: ${{ env.PAT }} + permission: write + issue-type: pull-request + repository: airbytehq/airbyte-cloud + dispatch-type: repository + commands: | + approve-and-merge + + - name: Edit comment with error message + if: steps.scd.outputs.error-message + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.comment.id }} + body: | + > Error: ${{ steps.scd.outputs.error-message }} From 55a32886a33713d9b505528407da8b6c880aba98 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Mon, 12 Dec 2022 14:32:34 -0500 Subject: [PATCH 002/134] CDK: `AbstractSource.read()` skips syncing stream if its unavailable (add `AvailabilityStrategy` concept) (#19977) * Rough first implememtation of AvailabilityStrategy s * Basic unit tests for AvailabilityStrategy and ScopedAvailabilityStrategy * Make availability_strategy a property, separate out tests * Remove from DeclarativeSource, remove Source parameter from methods, make default no AvailabilityStrategy * Add skip stream if not available to read() * Changes to CDK to get source-github working using AvailabilityStrategy, flakecheck * reorganize cdk class, add HTTPAvailabilityStrategy test * cleanup, docstrings * pull out error handling into separate method * Pass source and logger to check_connection method * Add documentation links, handle 403 specifically * Fix circular import * Add AvailabilityStrategy to Stream and HTTPStream classes * Remove AS from abstract_source, add to Stream, HTTPStream, AvailabilityStrategy unit tests passing for per-stream strategies * Modify MockHttpStream to set no AvailabilityStrategy since source test mocking doesn't support this * Move AvailabilityStrategy class to sources.streams * Move HTTPAvailabilityStrategy to http module * Use pascal case for HttpAvailabilityStrategy * Remove docs message method :( and default to True availability on unhandled HTTPErrors * add check_availability method to stream class * Add optional source parameter * Add test for connector-specific documentation, small tests refactor * Add test that performs the read() function for stream with default availability strategy * Add test for read function behavior when stream is unavailable * Add 403 info in logger message * Don't return error for other HTTPErrors * Split up error handling into methods 'unavailable_error_codes' and 'get_reason_for_error' * rework overrideable list of status codes to be a dict with reasons, to enforce that users provide reasons for all listed errors * Fix incorrect typing * Move HttpAvailability to its own module, fix flake errors * Fix ScopedAvailabilityStrategy, docstrings and types for streams/availability_strategy.py * Docstrings and types for core.py and http/availability_strategy.py * Move _get_stream_slices to a StreamHelper class * Docstrings + types for stream_helpers.py, cleanup test_availability.py * Clean up test_source.py * Move logic of getting the initial record from a stream to StreamHelper class * Add changelog and bump minor version * change 'is True' and 'is False' behavior * use mocker.MagicMock * Remove ScopedAvailabilityStrategy * Don't except non-403 errors, check_stream uses availability_strategy if possible * CDK: pass error to reasons_for_error_codes * make get_stream_slice public * Add tests for raising unhandled errors and retries are handled * Add tests for CheckStream via AvailabilityStrategy * Add documentation for stream availability of http streams * Move availability unit tests to correct modules, report error message if possible * Add test for reporting specific error if available --- airbyte-cdk/python/CHANGELOG.md | 3 + .../airbyte_cdk/sources/abstract_source.py | 6 +- .../declarative/checks/check_stream.py | 40 ++--- .../sources/declarative/declarative_source.py | 2 +- .../sources/streams/availability_strategy.py | 33 ++++ .../airbyte_cdk/sources/streams/core.py | 29 +++- .../streams/http/availability_strategy.py | 120 +++++++++++++ .../airbyte_cdk/sources/streams/http/http.py | 6 + .../sources/utils/stream_helpers.py | 44 +++++ .../python/docs/concepts/http-streams.md | 11 ++ airbyte-cdk/python/setup.py | 2 +- .../declarative/checks/test_check_stream.py | 59 +++++++ .../http/test_availability_strategy.py | 141 +++++++++++++++ .../streams/test_availability_strategy.py | 70 ++++++++ .../python/unit_tests/sources/test_source.py | 161 +++++++++++++++--- .../sources/utils/test_schema_helpers.py | 4 +- 16 files changed, 681 insertions(+), 50 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/streams/availability_strategy.py create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/streams/http/availability_strategy.py create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/utils/stream_helpers.py create mode 100644 airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py create mode 100644 airbyte-cdk/python/unit_tests/sources/streams/test_availability_strategy.py diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index f8824f945e71..12ddc2b20911 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.13.0 +Add `Stream.check_availability` and `Stream.AvailabilityStrategy`. Make `HttpAvailabilityStrategy` the default `HttpStream.AvailabilityStrategy`. + ## 0.12.4 Lookback window should applied when a state is supplied as well diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py index d1ac63e76e82..3f9efab8e48f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py @@ -106,6 +106,10 @@ def read( f"The requested stream {configured_stream.stream.name} was not found in the source." f" Available streams: {stream_instances.keys()}" ) + stream_is_available, error = stream_instance.check_availability(logger, self) + if not stream_is_available: + logger.warning(f"Skipped syncing stream '{stream_instance.name}' because it was unavailable. Error: {error}") + continue try: timer.start_event(f"Syncing stream {configured_stream.stream.name}") yield from self._read_stream( @@ -187,7 +191,7 @@ def _read_stream( @staticmethod def _limit_reached(internal_config: InternalConfig, records_counter: int) -> bool: """ - Check if record count reached liimt set by internal config. + Check if record count reached limit set by internal config. :param internal_config - internal CDK configuration separated from user defined config :records_counter - number of records already red :return True if limit reached, False otherwise diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py index c982f354f3b4..490cf104ec6d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py @@ -6,9 +6,9 @@ from dataclasses import InitVar, dataclass from typing import Any, List, Mapping, Tuple -from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker from airbyte_cdk.sources.source import Source +from airbyte_cdk.sources.utils.stream_helpers import StreamHelper from dataclasses_jsonschema import JsonSchemaMixin @@ -33,29 +33,19 @@ def check_connection(self, source: Source, logger: logging.Logger, config: Mappi if len(streams) == 0: return False, f"No streams to connect to from source {source}" for stream_name in self.stream_names: - if stream_name in stream_name_to_stream.keys(): - stream = stream_name_to_stream[stream_name] - try: - # Some streams need a stream slice to read records (eg if they have a SubstreamSlicer) - stream_slice = self._get_stream_slice(stream) - records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) - next(records) - except Exception as error: - return False, f"Unable to connect to stream {stream_name} - {error}" - else: + if stream_name not in stream_name_to_stream.keys(): raise ValueError(f"{stream_name} is not part of the catalog. Expected one of {stream_name_to_stream.keys()}") - return True, None - def _get_stream_slice(self, stream): - # We wrap the return output of stream_slices() because some implementations return types that are iterable, - # but not iterators such as lists or tuples - slices = iter( - stream.stream_slices( - cursor_field=stream.cursor_field, - sync_mode=SyncMode.full_refresh, - ) - ) - try: - return next(slices) - except StopIteration: - return {} + stream = stream_name_to_stream[stream_name] + try: + if stream.availability_strategy is not None: + stream_is_available, reason = stream.check_availability(logger, source) + if not stream_is_available: + return False, reason + else: + stream_helper = StreamHelper() + stream_helper.get_first_record(stream) + except Exception as error: + return False, f"Unable to connect to stream {stream_name} - {error}" + + return True, None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_source.py index 6e79356ee93b..beb3cfaa1d26 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_source.py @@ -17,7 +17,7 @@ class DeclarativeSource(AbstractSource): @property @abstractmethod def connection_checker(self) -> ConnectionChecker: - """Returns the ConnectioChecker to use for the `check` operation""" + """Returns the ConnectionChecker to use for the `check` operation""" def check_connection(self, logger, config) -> Tuple[bool, any]: """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/availability_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/availability_strategy.py new file mode 100644 index 000000000000..bb86a1c1de0d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/availability_strategy.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +import typing +from abc import ABC, abstractmethod +from typing import Optional, Tuple + +from airbyte_cdk.sources.streams import Stream + +if typing.TYPE_CHECKING: + from airbyte_cdk.sources import Source + + +class AvailabilityStrategy(ABC): + """ + Abstract base class for checking stream availability. + """ + + @abstractmethod + def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional["Source"]) -> Tuple[bool, Optional[str]]: + """ + Checks stream availability. + + :param stream: stream + :param logger: source logger + :param source: (optional) source + :return: A tuple of (boolean, str). If boolean is true, then the stream + is available, and no str is required. Otherwise, the stream is unavailable + for some reason and the str should describe what went wrong and how to + resolve the unavailability, if possible. + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py index d39c706eb9aa..5ff57550e003 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py @@ -5,9 +5,10 @@ import inspect import logging +import typing from abc import ABC, abstractmethod from functools import lru_cache -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import airbyte_cdk.sources.utils.casing as casing from airbyte_cdk.models import AirbyteLogMessage, AirbyteStream, AirbyteTraceMessage, SyncMode @@ -17,6 +18,10 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from deprecated.classic import deprecated +if typing.TYPE_CHECKING: + from airbyte_cdk.sources import Source + from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy + # A stream's read method can return one of the following types: # Mapping[str, Any]: The content of an AirbyteRecordMessage # AirbyteRecordMessage: An AirbyteRecordMessage @@ -170,6 +175,28 @@ def source_defined_cursor(self) -> bool: """ return True + def check_availability(self, logger: logging.Logger, source: Optional["Source"] = None) -> Tuple[bool, Optional[str]]: + """ + Checks whether this stream is available. + + :param logger: source logger + :param source: (optional) source + :return: A tuple of (boolean, str). If boolean is true, then this stream + is available, and no str is required. Otherwise, this stream is unavailable + for some reason and the str should describe what went wrong and how to + resolve the unavailability, if possible. + """ + if self.availability_strategy: + return self.availability_strategy.check_availability(self, logger, source) + return True, None + + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + """ + :return: The AvailabilityStrategy used to check whether this stream is available. + """ + return None + @property @abstractmethod def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/availability_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/availability_strategy.py new file mode 100644 index 000000000000..180e8dd3ee95 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/availability_strategy.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +import typing +from typing import Dict, Optional, Tuple + +import requests +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.utils.stream_helpers import StreamHelper +from requests import HTTPError + +if typing.TYPE_CHECKING: + from airbyte_cdk.sources import Source + + +class HttpAvailabilityStrategy(AvailabilityStrategy): + def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional["Source"]) -> Tuple[bool, Optional[str]]: + """ + Check stream availability by attempting to read the first record of the + stream. + + :param stream: stream + :param logger: source logger + :param source: (optional) source + :return: A tuple of (boolean, str). If boolean is true, then the stream + is available, and no str is required. Otherwise, the stream is unavailable + for some reason and the str should describe what went wrong and how to + resolve the unavailability, if possible. + """ + try: + stream_helper = StreamHelper() + stream_helper.get_first_record(stream) + except HTTPError as error: + return self.handle_http_error(stream, logger, source, error) + return True, None + + def handle_http_error( + self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError + ) -> Tuple[bool, Optional[str]]: + """ + Override this method to define error handling for various `HTTPError`s + that are raised while attempting to check a stream's availability. + + Checks whether an error's status_code is in a list of unavailable_error_codes, + and gets the associated reason for that error. + + :param stream: stream + :param logger: source logger + :param source: optional (source) + :param error: HTTPError raised while checking stream's availability. + :return: A tuple of (boolean, str). If boolean is true, then the stream + is available, and no str is required. Otherwise, the stream is unavailable + for some reason and the str should describe what went wrong and how to + resolve the unavailability, if possible. + """ + try: + status_code = error.response.status_code + reason = self.reasons_for_unavailable_status_codes(stream, logger, source, error)[status_code] + response_error_message = stream.parse_response_error_message(error.response) + if response_error_message: + reason += response_error_message + return False, reason + except KeyError: + # If the HTTPError is not in the dictionary of errors we know how to handle, don't except it + raise error + + def reasons_for_unavailable_status_codes( + self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError + ) -> Dict[int, str]: + """ + Returns a dictionary of HTTP status codes that indicate stream + unavailability and reasons explaining why a given status code may + have occurred and how the user can resolve that error, if applicable. + + :param stream: stream + :param logger: source logger + :param source: optional (source) + :return: A dictionary of (status code, reason) where the 'reason' explains + why 'status code' may have occurred and how the user can resolve that + error, if applicable. + """ + forbidden_error_message = f"The endpoint to access stream '{stream.name}' returned 403: Forbidden. " + forbidden_error_message += "This is most likely due to insufficient permissions on the credentials in use. " + forbidden_error_message += self._visit_docs_message(logger, source) + + reasons_for_codes: Dict[int, str] = {requests.codes.FORBIDDEN: forbidden_error_message} + return reasons_for_codes + + @staticmethod + def _visit_docs_message(logger: logging.Logger, source: Optional["Source"]) -> str: + """ + Creates a message indicicating where to look in the documentation for + more information on a given source by checking the spec of that source + (if provided) for a 'documentationUrl'. + + :param logger: source logger + :param source: optional (source) + :return: A message telling the user where to go to learn more about the source. + """ + if not source: + return "Please visit the connector's documentation to learn more. " + + try: + connector_spec = source.spec(logger) + docs_url = connector_spec.documentationUrl + if docs_url: + return f"Please visit {docs_url} to learn more. " + else: + return "Please visit the connector's documentation to learn more. " + + except FileNotFoundError: # If we are unit testing without implementing spec() method in source + if source: + docs_url = f"https://docs.airbyte.com/integrations/sources/{source.name}" + else: + docs_url = "https://docs.airbyte.com/integrations/sources/test" + + return f"Please visit {docs_url} to learn more." diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index 084eb052894e..35cef60807fe 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -13,7 +13,9 @@ import requests import requests_cache from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import Stream, StreamData +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from requests.auth import AuthBase from requests_cache.session import CachedSession @@ -113,6 +115,10 @@ def retry_factor(self) -> float: def authenticator(self) -> HttpAuthenticator: return self._authenticator + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return HttpAvailabilityStrategy() + @abstractmethod def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/stream_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/stream_helpers.py new file mode 100644 index 000000000000..6f972246693b --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/stream_helpers.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping, Optional + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.core import StreamData + + +class StreamHelper: + def get_first_record(self, stream: Stream) -> StreamData: + """ + Gets the first record for a stream. + + :param stream: stream + :return: StreamData containing the first record in the stream + """ + # Some streams need a stream slice to read records (e.g. if they have a SubstreamSlicer) + stream_slice = self.get_stream_slice(stream) + records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) + next(records) + + @staticmethod + def get_stream_slice(stream: Stream) -> Optional[Mapping[str, Any]]: + """ + Gets the first stream_slice from a given stream's stream_slices. + + :param stream: stream + :return: First stream slice from 'stream_slices' generator + """ + # We wrap the return output of stream_slices() because some implementations return types that are iterable, + # but not iterators such as lists or tuples + slices = iter( + stream.stream_slices( + cursor_field=stream.cursor_field, + sync_mode=SyncMode.full_refresh, + ) + ) + try: + return next(slices) + except StopIteration: + return {} diff --git a/airbyte-cdk/python/docs/concepts/http-streams.md b/airbyte-cdk/python/docs/concepts/http-streams.md index 5bedb787c747..0be02ce35cdc 100644 --- a/airbyte-cdk/python/docs/concepts/http-streams.md +++ b/airbyte-cdk/python/docs/concepts/http-streams.md @@ -81,3 +81,14 @@ When we are dealing with streams that depend on the results of another stream, w If you need to set any network-adapter keyword args on the outgoing HTTP requests such as `allow_redirects`, `stream`, `verify`, `cert`, etc.. override the `request_kwargs` method. Any option listed in [BaseAdapter.send](https://docs.python-requests.org/en/latest/api/#requests.adapters.BaseAdapter.send) can be returned as a keyword argument. + +## Stream Availability + +The CDK defines an `AvailabilityStrategy` for a stream, which is used to perform the `check_availability` method. This method checks whether +the stream is available before performing `read_records`. + +For HTTP streams, a default `HttpAvailabilityStrategy` is defined, which attempts to read the first record of the stream, and excepts +a dictionary of known error codes and associated reasons, `reasons_for_unavailable_status_codes`. By default, this list contains only +`requests.status_codes.FORBIDDEN` (403), with an associated error message that tells the user that they are likely missing permissions associated with that stream. + +You can override these known errors to except more error codes and inform the user how to resolve errors. diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 46f64877e0b0..0ac6109245a0 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.12.4", + version="0.13.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py index 10e43ea7c4d1..c7b84dc24b79 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py @@ -2,10 +2,15 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import logging +from typing import Any, Iterable, Mapping, Optional from unittest.mock import MagicMock import pytest +import requests from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy logger = None config = dict() @@ -27,6 +32,7 @@ def test_check_stream_with_slices_as_list(test_name, record, streams_to_check, stream_slice, expectation, slices_as_list): stream = MagicMock() stream.name = "s1" + stream.availability_strategy = None if slices_as_list: stream.stream_slices.return_value = [stream_slice] else: @@ -49,3 +55,56 @@ def test_check_stream_with_slices_as_list(test_name, record, streams_to_check, s def mock_read_records(responses, default_response=None, **kwargs): return lambda stream_slice, sync_mode: responses[frozenset(stream_slice)] if frozenset(stream_slice) in responses else default_response + + +@pytest.mark.parametrize( + "test_name, response_code, available_expectation, expected_messages", + [ + ("test_stream_unavailable_unhandled_error", 404, False, ["Unable to connect to stream mock_http_stream", "404 Client Error"]), + ("test_stream_unavailable_handled_error", 403, False, [ + "The endpoint to access stream 'mock_http_stream' returned 403: Forbidden.", + "This is most likely due to insufficient permissions on the credentials in use.", + ]), + ("test_stream_available", 200, True, []), + ], +) +def test_check_http_stream_via_availability_strategy(mocker, test_name, response_code, available_expectation, expected_messages): + class MockHttpStream(HttpStream): + url_base = "https://test_base_url.com" + primary_key = "" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.resp_counter = 1 + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + return "" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + stub_resp = {"data": self.resp_counter} + self.resp_counter += 1 + yield stub_resp + pass + + http_stream = MockHttpStream() + assert isinstance(http_stream, HttpStream) + assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) + + source = MagicMock() + source.streams.return_value = [http_stream] + + check_stream = CheckStream(stream_names=["mock_http_stream"], options={}) + + req = requests.Response() + req.status_code = response_code + mocker.patch.object(requests.Session, "send", return_value=req) + + logger = logging.getLogger(f"airbyte.{getattr(source, 'name', '')}") + stream_is_available, reason = check_stream.check_connection(source, logger, config) + + assert stream_is_available == available_expectation + for message in expected_messages: + assert message in reason diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py new file mode 100644 index 000000000000..1168dc609ca0 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Any, Iterable, List, Mapping, Optional, Tuple + +import pytest +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy +from airbyte_cdk.sources.streams.http.http import HttpStream +from requests import HTTPError + +logger = logging.getLogger("airbyte") + + +class MockHttpStream(HttpStream): + url_base = "https://test_base_url.com" + primary_key = "" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.resp_counter = 1 + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + return "" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + stub_resp = {"data": self.resp_counter} + self.resp_counter += 1 + yield stub_resp + pass + + def retry_factor(self) -> float: + return 0.01 + + +def test_default_http_availability_strategy(mocker): + http_stream = MockHttpStream() + assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) + + class MockResponse(requests.Response, mocker.MagicMock): + def __init__(self, *args, **kvargs): + mocker.MagicMock.__init__(self) + requests.Response.__init__(self, **kvargs) + self.json = mocker.MagicMock() + + response = MockResponse() + response.status_code = 403 + response.json.return_value = {"error": "Oh no!"} + mocker.patch.object(requests.Session, "send", return_value=response) + + stream_is_available, reason = http_stream.check_availability(logger) + assert not stream_is_available + + expected_messages = [ + "This is most likely due to insufficient permissions on the credentials in use.", + "Please visit the connector's documentation to learn more.", + "Oh no!", + ] + for message in expected_messages: + assert message in reason + + req = requests.Response() + req.status_code = 200 + mocker.patch.object(requests.Session, "send", return_value=req) + + stream_is_available, _ = http_stream.check_availability(logger) + assert stream_is_available + + +def test_http_availability_connector_specific_docs(mocker): + class MockSource(AbstractSource): + def __init__(self, streams: List[Stream] = None): + self._streams = streams + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + return True, "" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + if not self._streams: + raise Exception("Stream is not set") + return self._streams + + http_stream = MockHttpStream() + source = MockSource(streams=[http_stream]) + assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) + + req = requests.Response() + req.status_code = 403 + mocker.patch.object(requests.Session, "send", return_value=req, json={"error": "Oh no!"}) + + stream_is_available, reason = http_stream.check_availability(logger, source) + assert not stream_is_available + + expected_messages = [ + f"The endpoint to access stream '{http_stream.name}' returned 403: Forbidden.", + "This is most likely due to insufficient permissions on the credentials in use.", + f"Please visit https://docs.airbyte.com/integrations/sources/{source.name} to learn more.", + # "Oh no!", + ] + for message in expected_messages: + assert message in reason + + +def test_http_availability_raises_unhandled_error(mocker): + http_stream = MockHttpStream() + assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) + + req = requests.Response() + req.status_code = 404 + mocker.patch.object(requests.Session, "send", return_value=req) + + with pytest.raises(HTTPError): + http_stream.check_availability(logger) + + +def test_send_handles_retries_when_checking_availability(mocker, caplog): + http_stream = MockHttpStream() + assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) + + req_1 = requests.Response() + req_1.status_code = 429 + req_2 = requests.Response() + req_2.status_code = 503 + req_3 = requests.Response() + req_3.status_code = 200 + mock_send = mocker.patch.object(requests.Session, "send", side_effect=[req_1, req_2, req_3]) + + with caplog.at_level(logging.INFO): + stream_is_available, _ = http_stream.check_availability(logger) + + assert stream_is_available + assert mock_send.call_count == 3 + for message in ["Caught retryable error", "Response Code: 429", "Response Code: 503"]: + assert message in caplog.text diff --git a/airbyte-cdk/python/unit_tests/sources/streams/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/test_availability_strategy.py new file mode 100644 index 000000000000..277924b01197 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/test_availability_strategy.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.core import StreamData + +logger = logging.getLogger("airbyte") + + +class MockStream(Stream): + def __init__(self, name: str) -> Stream: + self._name = name + + @property + def name(self) -> str: + return self._name + + @property + def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: + pass + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[StreamData]: + pass + + +def test_no_availability_strategy(): + stream_1 = MockStream("stream") + assert stream_1.availability_strategy is None + + stream_1_is_available, _ = stream_1.check_availability(logger) + assert stream_1_is_available + + +def test_availability_strategy(): + class MockAvailabilityStrategy(AvailabilityStrategy): + def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional[Source]) -> Tuple[bool, any]: + if stream.name == "available_stream": + return True, None + return False, f"Could not reach stream '{stream.name}'." + + class MockStreamWithAvailabilityStrategy(MockStream): + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return MockAvailabilityStrategy() + + stream_1 = MockStreamWithAvailabilityStrategy("available_stream") + stream_2 = MockStreamWithAvailabilityStrategy("unavailable_stream") + + for stream in [stream_1, stream_2]: + assert isinstance(stream.availability_strategy, MockAvailabilityStrategy) + + stream_1_is_available, _ = stream_1.check_availability(logger) + assert stream_1_is_available + + stream_2_is_available, reason = stream_2.check_availability(logger) + assert not stream_2_is_available + assert "Could not reach stream 'unavailable_stream'" in reason diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index 5b67d57444eb..64a546108db1 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -7,10 +7,10 @@ import tempfile from collections import defaultdict from contextlib import nullcontext as does_not_raise -from typing import Any, List, Mapping, MutableMapping, Optional, Tuple -from unittest.mock import MagicMock +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import pytest +import requests from airbyte_cdk.models import ( AirbyteGlobalState, AirbyteStateBlob, @@ -24,6 +24,7 @@ ) from airbyte_cdk.sources import AbstractSource, Source from airbyte_cdk.sources.streams.core import Stream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from pydantic import ValidationError @@ -43,10 +44,15 @@ def discover(self, logger: logging.Logger, config: Mapping[str, Any]): class MockAbstractSource(AbstractSource): + def __init__(self, streams: Optional[List[Stream]] = None): + self._streams = streams + def check_connection(self, *args, **kwargs) -> Tuple[bool, Optional[Any]]: return True, "" def streams(self, *args, **kwargs) -> List[Stream]: + if self._streams: + return self._streams return [] @@ -79,26 +85,30 @@ def abstract_source(mocker): mocker.patch.multiple(HttpStream, __abstractmethods__=set()) mocker.patch.multiple(Stream, __abstractmethods__=set()) - class MockHttpStream(MagicMock, HttpStream): + class MockHttpStream(mocker.MagicMock, HttpStream): url_base = "http://example.com" path = "/dummy/path" - get_json_schema = MagicMock() + get_json_schema = mocker.MagicMock() def supports_incremental(self): return True def __init__(self, *args, **kvargs): - MagicMock.__init__(self) + mocker.MagicMock.__init__(self) HttpStream.__init__(self, *args, kvargs) - self.read_records = MagicMock() + self.read_records = mocker.MagicMock() + + @property + def availability_strategy(self): + return None - class MockStream(MagicMock, Stream): + class MockStream(mocker.MagicMock, Stream): page_size = None - get_json_schema = MagicMock() + get_json_schema = mocker.MagicMock() - def __init__(self, *args, **kvargs): - MagicMock.__init__(self) - self.read_records = MagicMock() + def __init__(self, **kwargs): + mocker.MagicMock.__init__(self) + self.read_records = mocker.MagicMock() streams = [MockHttpStream(), MockStream()] @@ -385,8 +395,8 @@ def test_internal_config(abstract_source, catalog): assert not non_http_stream.page_size -def test_internal_config_limit(abstract_source, catalog): - logger_mock = MagicMock() +def test_internal_config_limit(mocker, abstract_source, catalog): + logger_mock = mocker.MagicMock() logger_mock.level = logging.DEBUG del catalog.streams[1] STREAM_LIMIT = 2 @@ -423,8 +433,8 @@ def test_internal_config_limit(abstract_source, catalog): SCHEMA = {"type": "object", "properties": {"value": {"type": "string"}}} -def test_source_config_no_transform(abstract_source, catalog): - logger_mock = MagicMock() +def test_source_config_no_transform(mocker, abstract_source, catalog): + logger_mock = mocker.MagicMock() logger_mock.level = logging.DEBUG streams = abstract_source.streams(None) http_stream, non_http_stream = streams @@ -437,8 +447,8 @@ def test_source_config_no_transform(abstract_source, catalog): assert non_http_stream.get_json_schema.call_count == 5 -def test_source_config_transform(abstract_source, catalog): - logger_mock = MagicMock() +def test_source_config_transform(mocker, abstract_source, catalog): + logger_mock = mocker.MagicMock() logger_mock.level = logging.DEBUG streams = abstract_source.streams(None) http_stream, non_http_stream = streams @@ -451,8 +461,8 @@ def test_source_config_transform(abstract_source, catalog): assert [r.record.data for r in records] == [{"value": "23"}] * 2 -def test_source_config_transform_and_no_transform(abstract_source, catalog): - logger_mock = MagicMock() +def test_source_config_transform_and_no_transform(mocker, abstract_source, catalog): + logger_mock = mocker.MagicMock() logger_mock.level = logging.DEBUG streams = abstract_source.streams(None) http_stream, non_http_stream = streams @@ -462,3 +472,116 @@ def test_source_config_transform_and_no_transform(abstract_source, catalog): records = [r for r in abstract_source.read(logger=logger_mock, config={}, catalog=catalog, state={})] assert len(records) == 2 assert [r.record.data for r in records] == [{"value": "23"}, {"value": 23}] + + +def test_read_default_http_availability_strategy_stream_available(catalog, mocker): + mocker.patch.multiple(HttpStream, __abstractmethods__=set()) + mocker.patch.multiple(Stream, __abstractmethods__=set()) + + class MockHttpStream(mocker.MagicMock, HttpStream): + url_base = "http://example.com" + path = "/dummy/path" + get_json_schema = mocker.MagicMock() + + def supports_incremental(self): + return True + + def __init__(self, *args, **kvargs): + mocker.MagicMock.__init__(self) + HttpStream.__init__(self, *args, kvargs) + self.read_records = mocker.MagicMock() + + class MockStream(mocker.MagicMock, Stream): + page_size = None + get_json_schema = mocker.MagicMock() + + def __init__(self, *args, **kvargs): + mocker.MagicMock.__init__(self) + self.read_records = mocker.MagicMock() + + streams = [MockHttpStream(), MockStream()] + http_stream, non_http_stream = streams + assert isinstance(http_stream, HttpStream) + assert not isinstance(non_http_stream, HttpStream) + + assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) + assert non_http_stream.availability_strategy is None + + # Add an extra record for the default HttpAvailabilityStrategy to pull from + # during the try: next(records) check, since we are mocking the return value + # and not re-creating the generator like we would during actual reading + http_stream.read_records.return_value = iter([{"value": "test"}] + [{}] * 3) + non_http_stream.read_records.return_value = iter([{}] * 3) + + source = MockAbstractSource(streams=streams) + logger = logging.getLogger(f"airbyte.{getattr(abstract_source, 'name', '')}") + records = [r for r in source.read(logger=logger, config={}, catalog=catalog, state={})] + # 3 for http stream and 3 for non http stream + assert len(records) == 3 + 3 + assert http_stream.read_records.called + assert non_http_stream.read_records.called + + +def test_read_default_http_availability_strategy_stream_unavailable(catalog, mocker, caplog): + mocker.patch.multiple(Stream, __abstractmethods__=set()) + + class MockHttpStream(HttpStream): + url_base = "https://test_base_url.com" + primary_key = "" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.resp_counter = 1 + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + return "" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + stub_response = {"data": self.resp_counter} + self.resp_counter += 1 + yield stub_response + + class MockStream(mocker.MagicMock, Stream): + page_size = None + get_json_schema = mocker.MagicMock() + + def __init__(self, *args, **kvargs): + mocker.MagicMock.__init__(self) + self.read_records = mocker.MagicMock() + + streams = [MockHttpStream(), MockStream()] + http_stream, non_http_stream = streams + assert isinstance(http_stream, HttpStream) + assert not isinstance(non_http_stream, HttpStream) + + assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) + assert non_http_stream.availability_strategy is None + + # Don't set anything for read_records return value for HttpStream, since + # it should be skipped due to the stream being unavailable + non_http_stream.read_records.return_value = iter([{}] * 3) + + # Patch HTTP request to stream endpoint to make it unavailable + req = requests.Response() + req.status_code = 403 + mocker.patch.object(requests.Session, "send", return_value=req) + + source = MockAbstractSource(streams=streams) + logger = logging.getLogger("test_read_default_http_availability_strategy_stream_unavailable") + with caplog.at_level(logging.WARNING): + records = [r for r in source.read(logger=logger, config={}, catalog=catalog, state={})] + + # 0 for http stream and 3 for non http stream + assert len(records) == 0 + 3 + assert non_http_stream.read_records.called + expected_logs = [ + f"Skipped syncing stream '{http_stream.name}' because it was unavailable.", + f"The endpoint to access stream '{http_stream.name}' returned 403: Forbidden.", + "This is most likely due to insufficient permissions on the credentials in use.", + f"Please visit https://docs.airbyte.com/integrations/sources/{source.name} to learn more." + ] + for message in expected_logs: + assert message in caplog.text diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py index 2274ffb02ae5..38aa713a5e78 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py @@ -4,6 +4,7 @@ import json +import logging import os import shutil import sys @@ -12,14 +13,13 @@ from pathlib import Path import jsonref -from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification, FailureType from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader, check_config_against_spec_or_exit from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pytest import fixture from pytest import raises as pytest_raises -logger = AirbyteLogger() +logger = logging.getLogger("airbyte") MODULE = sys.modules[__name__] From 5f4d43d5cf92f84bcc938328bfa7287d9244365c Mon Sep 17 00:00:00 2001 From: Mark Berger Date: Mon, 12 Dec 2022 22:07:45 +0200 Subject: [PATCH 003/134] Create documentation for Destination namespace example settings (#20255) - Added "Learn more" link to documentation namespace --- .../DestinationNamespaceModal/DestinationNamespaceModal.tsx | 6 ++++++ airbyte-webapp/src/locales/en.json | 1 + 2 files changed, 7 insertions(+) diff --git a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx index 48587c2e580f..d72ee5b0d31b 100644 --- a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx +++ b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx @@ -10,6 +10,7 @@ import { ModalBody, ModalFooter } from "components/ui/Modal"; import { Text } from "components/ui/Text"; import { NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { links } from "utils/links"; import { FormikConnectionFormValues } from "views/Connection/ConnectionForm/formConfig"; import styles from "./DestinationNamespaceModal.module.scss"; @@ -143,6 +144,11 @@ export const DestinationNamespaceModal: React.FC + + + + + diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index bcec3f1ca986..2a4cf7201265 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -171,6 +171,7 @@ "connectionForm.modal.destinationNamespace.table.data.exampleSourceNamespace": "\"$SOURCE_NAMESPACE\"", "connectionForm.modal.destinationNamespace.table.data.exampleMySourceNamespace": "\"my_$SOURCE_NAMESPACE_schema\"", "connectionForm.modal.destinationNamespace.table.data.custom": "custom", + "connectionForm.modal.destinationNamespace.learnMore.link": "Learn more", "connectionForm.modal.destinationStreamNames.title": "Destination stream names", "connectionForm.modal.destinationStreamNames.radioButton.mirror": "Mirror source name", From 2f306ea1d0eba625c86b53a669aecad5a06338d8 Mon Sep 17 00:00:00 2001 From: darynaishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Mon, 12 Dec 2022 22:45:18 +0200 Subject: [PATCH 004/134] Low-Code: added SessionTokenAuthenticator (#19716) * added SessionTokenAuthenticator for low-code * added doc comments and formatted files * fixed doc strings * added login_url and validate_session_url to config * removed unused f-string, formatted * Update log Co-authored-by: Serhii Lazebnyi --- airbyte-cdk/python/CHANGELOG.md | 3 + .../sources/declarative/auth/token.py | 125 ++++++++++++ .../declarative/config_component_schema.json | 72 +++++++ .../parsers/class_types_registry.py | 8 +- airbyte-cdk/python/setup.py | 3 +- .../auth/test_session_token_auth.py | 184 ++++++++++++++++++ 6 files changed, 393 insertions(+), 2 deletions(-) create mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/auth/test_session_token_auth.py diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 12ddc2b20911..116c818a2e09 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.13.1 +Low-code: Add `SessionTokenAuthenticator` + ## 0.13.0 Add `Stream.check_availability` and `Stream.AvailabilityStrategy`. Make `HttpAvailabilityStrategy` the default `HttpStream.AvailabilityStrategy`. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py index c37e46e904ef..a4f105dbfe60 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -3,13 +3,16 @@ # import base64 +import logging from dataclasses import InitVar, dataclass from typing import Any, Mapping, Union +import requests from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Config from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator +from cachetools import TTLCache, cached from dataclasses_jsonschema import JsonSchemaMixin @@ -115,3 +118,125 @@ def token(self) -> str: auth_string = f"{self._username.eval(self.config)}:{self._password.eval(self.config)}".encode("utf8") b64_encoded = base64.b64encode(auth_string).decode("utf8") return f"Basic {b64_encoded}" + + +""" + maxsize - The maximum size of the cache + ttl - time-to-live value in seconds + docs https://cachetools.readthedocs.io/en/latest/ + maxsize=1000 - when the cache is full, in this case more than 1000, + i.e. by adding another item the cache would exceed its maximum size, the cache must choose which item(s) to discard + ttl=86400 means that cached token will live for 86400 seconds (one day) +""" +cacheSessionTokenAuthenticator = TTLCache(maxsize=1000, ttl=86400) + + +@cached(cacheSessionTokenAuthenticator) +def get_new_session_token(api_url: str, username: str, password: str, response_key: str) -> str: + """ + This method retrieves session token from api by username and password for SessionTokenAuthenticator. + It's cashed to avoid a multiple calling by sync and updating session token every stream sync. + Args: + api_url: api url for getting new session token + username: username for auth + password: password for auth + response_key: field name in response to retrieve a session token + + Returns: + session token + """ + response = requests.post( + f"{api_url}", + headers={"Content-Type": "application/json"}, + json={"username": username, "password": password}, + ) + response.raise_for_status() + if not response.ok: + raise ConnectionError(f"Failed to retrieve new session token, response code {response.status_code} because {response.reason}") + return response.json()[response_key] + + +@dataclass +class SessionTokenAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator, JsonSchemaMixin): + """ + Builds auth based on session tokens. + A session token is a random value generated by a server to identify + a specific user for the duration of one interaction session. + + The header is of the form + `"Specific Header": "Session Token Value"` + + Attributes: + api_url (Union[InterpolatedString, str]): Base api url of source + username (Union[InterpolatedString, str]): The username + config (Config): The user-provided configuration as specified by the source's spec + password (Union[InterpolatedString, str]): The password + header (Union[InterpolatedString, str]): Specific header of source for providing session token + options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation + session_token (Union[InterpolatedString, str]): Session token generated by user + session_token_response_key (Union[InterpolatedString, str]): Key for retrieving session token from api response + login_url (Union[InterpolatedString, str]): Url fot getting a specific session token + validate_session_url (Union[InterpolatedString, str]): Url to validate passed session token + """ + + api_url: Union[InterpolatedString, str] + header: Union[InterpolatedString, str] + session_token: Union[InterpolatedString, str] + session_token_response_key: Union[InterpolatedString, str] + username: Union[InterpolatedString, str] + config: Config + options: InitVar[Mapping[str, Any]] + login_url: Union[InterpolatedString, str] + validate_session_url: Union[InterpolatedString, str] + password: Union[InterpolatedString, str] = "" + + def __post_init__(self, options): + self._username = InterpolatedString.create(self.username, options=options) + self._password = InterpolatedString.create(self.password, options=options) + self._api_url = InterpolatedString.create(self.api_url, options=options) + self._header = InterpolatedString.create(self.header, options=options) + self._session_token = InterpolatedString.create(self.session_token, options=options) + self._session_token_response_key = InterpolatedString.create(self.session_token_response_key, options=options) + self._login_url = InterpolatedString.create(self.login_url, options=options) + self._validate_session_url = InterpolatedString.create(self.validate_session_url, options=options) + + self.logger = logging.getLogger("airbyte") + + @property + def auth_header(self) -> str: + return self._header.eval(self.config) + + @property + def token(self) -> str: + if self._session_token.eval(self.config): + if self.is_valid_session_token(): + return self._session_token.eval(self.config) + if self._password.eval(self.config) and self._username.eval(self.config): + username = self._username.eval(self.config) + password = self._password.eval(self.config) + session_token_response_key = self._session_token_response_key.eval(self.config) + api_url = f"{self._api_url.eval(self.config)}{self._login_url.eval(self.config)}" + + self.logger.info("Using generated session token by username and password") + return get_new_session_token(api_url, username, password, session_token_response_key) + + raise ConnectionError("Invalid credentials: session token is not valid or provide username and password") + + def is_valid_session_token(self) -> bool: + try: + response = requests.get( + f"{self._api_url.eval(self.config)}{self._validate_session_url.eval(self.config)}", + headers={self.auth_header: self._session_token.eval(self.config)}, + ) + response.raise_for_status() + except requests.exceptions.HTTPError as e: + if e.response.status_code == requests.codes["unauthorized"]: + self.logger.info(f"Unable to connect by session token from config due to {str(e)}") + return False + else: + raise ConnectionError(f"Error while validating session token: {e}") + if response.ok: + self.logger.info("Connection check for source is successful.") + return True + else: + raise ConnectionError(f"Failed to retrieve new session token, response code {response.status_code} because {response.reason}") diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json index 8f61c0716113..607cfacd930b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json @@ -289,6 +289,9 @@ }, { "$ref": "#/definitions/BasicHttpAuthenticator" + }, + { + "$ref": "#/definitions/SessionTokenAuthenticator" } ] }, @@ -629,6 +632,75 @@ ], "description": "\n Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using base64\n https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme\n\n The header is of the form\n `\"Authorization\": \"Basic \"`\n\n Attributes:\n username (Union[InterpolatedString, str]): The username\n config (Config): The user-provided configuration as specified by the source's spec\n password (Union[InterpolatedString, str]): The password\n options (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation\n " }, + "SessionTokenAuthenticator": { + "allOf": [ + { + "$ref": "#/definitions/DeclarativeAuthenticator" + }, + { + "type": "object", + "required": ["api_url", "header"], + "properties": { + "api_url": { + "anyOf": [ + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } + ] + }, + "session_token": { + "anyOf": [ + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } + ], + "default": "" + }, + "username": { + "anyOf": [ + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } + ], + "default": "" + }, + "password": { + "anyOf": [ + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } + ], + "default": "" + }, + "header": { + "anyOf": [ + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } + ] + }, + "config": { + "type": "object" + } + } + } + ] + }, "CompositeErrorHandler": { "allOf": [ { diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index 66d4d6b0bc90..f26287fa671f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -6,7 +6,12 @@ from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator -from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator +from airbyte_cdk.sources.declarative.auth.token import ( + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + SessionTokenAuthenticator, +) from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor @@ -78,6 +83,7 @@ "SingleSlice": SingleSlice, "Spec": Spec, "SubstreamSlicer": SubstreamSlicer, + "SessionTokenAuthenticator": SessionTokenAuthenticator, "WaitUntilTimeFromHeader": WaitUntilTimeFromHeaderBackoffStrategy, "WaitTimeFromHeader": WaitTimeFromHeaderBackoffStrategy, } diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 0ac6109245a0..c4065d6fa8e1 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.13.0", + version="0.13.1", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -58,6 +58,7 @@ "requests_cache", "Deprecated~=1.2", "Jinja2~=3.1.2", + "cachetools", ], python_requires=">=3.9", extras_require={ diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_session_token_auth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_session_token_auth.py new file mode 100644 index 000000000000..47334cbdfafd --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_session_token_auth.py @@ -0,0 +1,184 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.declarative.auth.token import SessionTokenAuthenticator, get_new_session_token +from requests.exceptions import HTTPError + +options = {"hello": "world"} +instance_api_url = "https://airbyte.metabaseapp.com/api/" +username = "username" +password = "password" +session_token = "session_token" +header = "X-App-Session" +session_token_response_key = "id" +login_url = "session" +validate_session_url = "user/current" + +input_instance_api_url = "{{ config['instance_api_url'] }}" +input_username = "{{ config['username'] }}" +input_password = "{{ config['password'] }}" +input_session_token = "{{ config['session_token'] }}" + +config = { + "instance_api_url": instance_api_url, + "username": username, + "password": password, + "session_token": session_token, + "header": header, + "session_token_response_key": session_token_response_key, + "login_url": login_url, + "validate_session_url": validate_session_url +} + +config_session_token = { + "instance_api_url": instance_api_url, + "username": "", + "password": "", + "session_token": session_token, + "header": header, + "session_token_response_key": session_token_response_key, + "login_url": login_url, + "validate_session_url": validate_session_url +} + +config_username_password = { + "instance_api_url": instance_api_url, + "username": username, + "password": password, + "session_token": "", + "header": header, + "session_token_response_key": session_token_response_key, + "login_url": login_url, + "validate_session_url": validate_session_url +} + + +def test_auth_header(): + auth_header = SessionTokenAuthenticator( + config=config, + options=options, + api_url=input_instance_api_url, + username=input_username, + password=input_password, + session_token=input_session_token, + header=header, + session_token_response_key=session_token_response_key, + login_url=login_url, + validate_session_url=validate_session_url + ).auth_header + assert auth_header == "X-App-Session" + + +def test_get_token_valid_session(requests_mock): + requests_mock.get( + f"{config_session_token['instance_api_url']}user/current", + json={"common_name": "common_name", "last_login": "last_login"} + ) + + token = SessionTokenAuthenticator( + config=config_session_token, + options=options, + api_url=input_instance_api_url, + username=input_username, + password=input_password, + session_token=input_session_token, + header=header, + session_token_response_key=session_token_response_key, + login_url=login_url, + validate_session_url=validate_session_url + ).token + assert token == "session_token" + + +def test_get_token_invalid_session_unauthorized(): + with pytest.raises(ConnectionError): + _ = SessionTokenAuthenticator( + config=config_session_token, + options=options, + api_url=input_instance_api_url, + username=input_username, + password=input_password, + session_token=input_session_token, + header=header, + session_token_response_key=session_token_response_key, + login_url=login_url, + validate_session_url=validate_session_url + ).token + + +def test_get_token_invalid_username_password_unauthorized(): + with pytest.raises(HTTPError): + _ = SessionTokenAuthenticator( + config=config_username_password, + options=options, + api_url=input_instance_api_url, + username=input_username, + password=input_password, + session_token=input_session_token, + header=header, + session_token_response_key=session_token_response_key, + validate_session_url=validate_session_url, + login_url=login_url + ).token + + +def test_get_token_username_password(requests_mock): + requests_mock.post(f"{config['instance_api_url']}session", json={"id": "some session id"}) + + token = SessionTokenAuthenticator( + config=config_username_password, + options=options, + api_url=input_instance_api_url, + username=input_username, + password=input_password, + session_token=input_session_token, + header=header, + session_token_response_key=session_token_response_key, + login_url=login_url, + validate_session_url=validate_session_url + ).token + assert token == "some session id" + + +def test_check_is_valid_session_token(requests_mock): + requests_mock.get(f"{config['instance_api_url']}user/current", + json={"common_name": "common_name", "last_login": "last_login"}) + + assert SessionTokenAuthenticator( + config=config, + options=options, + api_url=input_instance_api_url, + username=input_username, + password=input_password, + session_token=input_session_token, + header=header, + session_token_response_key=session_token_response_key, + validate_session_url=validate_session_url, + login_url=login_url + ).is_valid_session_token() + + +def test_check_is_valid_session_token_unauthorized(): + assert not SessionTokenAuthenticator( + config=config, + options=options, + api_url=input_instance_api_url, + username=input_username, + password=input_password, + session_token=input_session_token, + header=header, + session_token_response_key=session_token_response_key, + login_url=login_url, + validate_session_url=validate_session_url + ).is_valid_session_token() + + +def test_get_new_session_token(requests_mock): + requests_mock.post(f"{config['instance_api_url']}session", headers={"Content-Type": "application/json"}, + json={"id": "some session id"}) + + session_token = get_new_session_token(f'{config["instance_api_url"]}session', config["username"], + config["password"], config["session_token_response_key"]) + assert session_token == "some session id" From 7afc6896789657970279dde1dcbc009d675eb4c3 Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Mon, 12 Dec 2022 13:16:40 -0800 Subject: [PATCH 005/134] [Connector Builder] Configuration UI MVP (#20008) * move connector builder components into the same shared components/connectorBuilder directory * move diff over from poc branch * save current progress * add modal for adding streams * focus stream after adding and reset button style * add reset confirm modal and select view on add * style global config and streams buttons * styling improvements * handle long stream names better * pull in connector manifest schema directly * add box shadows to resizable panels * upgrade orval and use connector manifest schema directly * remove airbyte protocol from connector builder api spec * generate python models from openapi change * fix position of yaml toggle * handle no stream case with better looking message * group global fields into single object and fix console error * confirmation modal on toggling dirty form + cleanup * fix connector name display * undo change to manifest schema * remove commented code * remove unnecessary change * fix spacing * use shadow mixin for connector img * add comment about connector img * change onSubmit to no-op * remove console log * clean up styling * simplify sidebar to remove StreamSelectButton component * swap colors of toggle * move FormikPatch to src/core/form * move types up to connectorBuilder/ level * use grid display for ui yaml toggle button * use spread instead of setting array index directly * add intl in missing places * pull connector manifest schema in through separate openapi spec * use correct intl string id * throttle setting json manifest in yaml editor * use button prop instead of manually styling * consolidate AddStreamButton styles * fix sidebar flex styles * use specific flex properties instead of flex * clean up download and reset button styles * use row-reverse for yaml editor download button * fix stream selector styles to remove margins * give connector setup guide panel same corner and shadow styles * remove blur from page display * set view to stream when selected in test panel * add placeholder when stream name is empty * switch to index-based stream selection to preserve testing panel selected stream on rename * handle empty name in stream selector --- airbyte-webapp/.gitignore | 1 + airbyte-webapp/orval.config.ts | 17 +++ .../public/images/octavia/pointing.svg | 52 +++++++ .../Builder/AddStreamButton.module.scss | 20 +++ .../Builder/AddStreamButton.tsx | 101 ++++++++++++++ .../Builder/Builder.module.scss | 24 ++++ .../connectorBuilder/Builder/Builder.tsx | 31 ++++ .../Builder/BuilderCard.module.scss | 8 ++ .../connectorBuilder/Builder/BuilderCard.tsx | 9 ++ .../Builder/BuilderField.module.scss | 7 + .../connectorBuilder/Builder/BuilderField.tsx | 98 +++++++++++++ .../Builder/BuilderSidebar.module.scss | 104 ++++++++++++++ .../Builder/BuilderSidebar.tsx | 132 ++++++++++++++++++ .../Builder/GlobalConfigView.tsx | 21 +++ .../Builder/StreamConfigView.tsx | 36 +++++ .../Builder/UiYamlToggleButton.module.scss | 38 +++++ .../Builder/UiYamlToggleButton.tsx | 39 ++++++ .../{YamlEditor => }/DownloadYamlButton.tsx | 18 ++- .../PageDisplay.module.scss | 13 -- .../StreamSelector.module.scss | 13 +- .../StreamTestingPanel/StreamSelector.tsx | 32 +++-- .../StreamTester.module.scss | 5 +- .../StreamTestingPanel/StreamTester.tsx | 15 +- .../StreamTestingPanel.module.scss | 18 ++- .../StreamTestingPanel/StreamTestingPanel.tsx | 31 ++-- .../YamlEditor/YamlEditor.module.scss | 7 +- .../YamlEditor/YamlEditor.tsx | 64 +++++++-- .../src/components/connectorBuilder/types.ts | 50 +++++++ .../components/ui/ListBox/ListBox.module.scss | 1 + .../ui/ResizablePanels/ResizablePanels.tsx | 12 +- airbyte-webapp/src/core/form/FormikPatch.ts | 29 ++++ airbyte-webapp/src/locales/en.json | 20 +++ .../ConnectorBuilderPage.module.scss | 9 +- .../ConnectorBuilderPage.tsx | 62 +++++--- airbyte-webapp/src/pages/routes.tsx | 2 +- airbyte-webapp/src/scss/_mixins.scss | 8 ++ .../ConnectorBuilderStateService.tsx | 107 +++++++++----- .../connector_manifest_openapi.yaml | 9 ++ .../ConnectorDocumentationLayout.module.scss | 8 ++ .../ConnectorDocumentationLayout.tsx | 1 + .../Connector/ConnectorForm/ConnectorForm.tsx | 8 +- .../Connector/ConnectorForm/useBuildForm.tsx | 28 +--- 42 files changed, 1132 insertions(+), 176 deletions(-) create mode 100644 airbyte-webapp/public/images/octavia/pointing.svg create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss create mode 100644 airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx rename airbyte-webapp/src/components/connectorBuilder/{YamlEditor => }/DownloadYamlButton.tsx (79%) create mode 100644 airbyte-webapp/src/components/connectorBuilder/types.ts create mode 100644 airbyte-webapp/src/core/form/FormikPatch.ts create mode 100644 airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml diff --git a/airbyte-webapp/.gitignore b/airbyte-webapp/.gitignore index 8e572d98ba49..4c2aede53775 100644 --- a/airbyte-webapp/.gitignore +++ b/airbyte-webapp/.gitignore @@ -32,3 +32,4 @@ storybook-static/ # Ignore generated API clients, since they're automatically generated /src/core/request/AirbyteClient.ts /src/core/request/ConnectorBuilderClient.ts +/src/core/request/ConnectorManifest.ts diff --git a/airbyte-webapp/orval.config.ts b/airbyte-webapp/orval.config.ts index f859eaa5e4e2..f32b8e5fa769 100644 --- a/airbyte-webapp/orval.config.ts +++ b/airbyte-webapp/orval.config.ts @@ -43,4 +43,21 @@ export default defineConfig({ }, }, }, + connectorManifest: { + input: "./src/services/connectorBuilder/connector_manifest_openapi.yaml", + output: { + target: "./src/core/request/ConnectorManifest.ts", + prettier: true, + override: { + header: (info) => [ + `eslint-disable`, + `Generated by orval 🍺`, + `Do not edit manually. Run "npm run generate-client" instead.`, + ...(info.title ? [info.title] : []), + ...(info.description ? [info.description] : []), + ...(info.version ? [`OpenAPI spec version: ${info.version}`] : []), + ], + }, + }, + }, }); diff --git a/airbyte-webapp/public/images/octavia/pointing.svg b/airbyte-webapp/public/images/octavia/pointing.svg new file mode 100644 index 000000000000..b4453c1fb31d --- /dev/null +++ b/airbyte-webapp/public/images/octavia/pointing.svg @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss new file mode 100644 index 000000000000..6c1f9c41b990 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.module.scss @@ -0,0 +1,20 @@ +@use "scss/variables"; +@use "scss/colors"; + +$buttonWidth: 26px; + +.body { + display: flex; + flex-direction: column; + gap: variables.$spacing-xl; +} + +.addButton { + width: $buttonWidth; + height: $buttonWidth !important; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + padding: 9px !important; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx new file mode 100644 index 000000000000..2177db82409c --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/AddStreamButton.tsx @@ -0,0 +1,101 @@ +import { Form, Formik, useField } from "formik"; +import { useState } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; + +import { Button } from "components/ui/Button"; +import { Modal, ModalBody, ModalFooter } from "components/ui/Modal"; + +import { FormikPatch } from "core/form/FormikPatch"; + +import { ReactComponent as PlusIcon } from "../../connection/ConnectionOnboarding/plusIcon.svg"; +import { BuilderStream } from "../types"; +import styles from "./AddStreamButton.module.scss"; +import { BuilderField } from "./BuilderField"; + +interface AddStreamValues { + streamName: string; + urlPath: string; +} + +interface AddStreamButtonProps { + onAddStream: (addedStreamNum: number) => void; +} + +export const AddStreamButton: React.FC = ({ onAddStream }) => { + const { formatMessage } = useIntl(); + const [isOpen, setIsOpen] = useState(false); + const [streamsField, , helpers] = useField("streams"); + const numStreams = streamsField.value.length; + + return ( + <> + + + + + + + + )} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss new file mode 100644 index 000000000000..108a100d0dfb --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.module.scss @@ -0,0 +1,24 @@ +@use "scss/variables"; +@use "scss/colors"; +@use "scss/mixins"; + +.container { + height: 100%; + display: flex; +} + +.sidebar { + @include mixins.right-shadow; + + width: 200px; + flex: 0 0 auto; + background-color: colors.$white; +} + +.form { + flex: 1; + padding: variables.$spacing-xl; + display: flex; + flex-direction: column; + gap: variables.$spacing-xl; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx new file mode 100644 index 000000000000..fb40a24a2d9f --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/Builder.tsx @@ -0,0 +1,31 @@ +import { Form } from "formik"; +import { useEffect } from "react"; + +import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; + +import { BuilderFormValues } from "../types"; +import styles from "./Builder.module.scss"; +import { BuilderSidebar } from "./BuilderSidebar"; +import { GlobalConfigView } from "./GlobalConfigView"; +import { StreamConfigView } from "./StreamConfigView"; + +interface BuilderProps { + values: BuilderFormValues; + toggleYamlEditor: () => void; +} + +export const Builder: React.FC = ({ values, toggleYamlEditor }) => { + const { setBuilderFormValues, selectedView } = useConnectorBuilderState(); + useEffect(() => { + setBuilderFormValues(values); + }, [values, setBuilderFormValues]); + + return ( +
+ +
+ {selectedView === "global" ? : } + +
+ ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss new file mode 100644 index 000000000000..48fedff9a2a0 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.module.scss @@ -0,0 +1,8 @@ +@use "scss/variables"; + +.card { + padding: variables.$spacing-xl; + display: flex; + flex-direction: column; + gap: variables.$spacing-xl; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx new file mode 100644 index 000000000000..9bbb968a3d29 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderCard.tsx @@ -0,0 +1,9 @@ +import React from "react"; + +import { Card } from "components/ui/Card"; + +import styles from "./BuilderCard.module.scss"; + +export const BuilderCard: React.FC> = ({ children }) => { + return {children}; +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss new file mode 100644 index 000000000000..f198f89561f5 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.module.scss @@ -0,0 +1,7 @@ +@use "scss/variables"; +@use "scss/colors"; + +.error { + margin-top: variables.$spacing-sm; + color: colors.$red; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx new file mode 100644 index 000000000000..bd799ff8efed --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderField.tsx @@ -0,0 +1,98 @@ +import { useField } from "formik"; +import { FormattedMessage } from "react-intl"; +import * as yup from "yup"; + +import { ControlLabels } from "components/LabeledControl"; +import { DropDown } from "components/ui/DropDown"; +import { Input } from "components/ui/Input"; +import { TagInput } from "components/ui/TagInput"; +import { Text } from "components/ui/Text"; + +import styles from "./BuilderField.module.scss"; + +interface EnumFieldProps { + options: string[]; + value: string; + setValue: (value: string) => void; + error: boolean; +} + +interface ArrayFieldProps { + name: string; + value: string[]; + setValue: (value: string[]) => void; + error: boolean; +} + +interface BaseFieldProps { + // path to the location in the Connector Manifest schema which should be set by this component + path: string; + label: string; + tooltip?: string; + optional?: boolean; +} + +type BuilderFieldProps = BaseFieldProps & ({ type: "text" } | { type: "array" } | { type: "enum"; options: string[] }); + +const EnumField: React.FC = ({ options, value, setValue, error, ...props }) => { + return ( + { + return { label: option, value: option }; + })} + onChange={(selected) => selected && setValue(selected.value)} + value={value} + error={error} + /> + ); +}; + +const ArrayField: React.FC = ({ name, value, setValue, error }) => { + return setValue(value)} error={error} />; +}; + +export const BuilderField: React.FC = ({ path, label, tooltip, optional = false, ...props }) => { + let yupSchema = props.type === "array" ? yup.array().of(yup.string()) : yup.string(); + if (!optional) { + yupSchema = yupSchema.required("form.empty.error"); + } + const fieldConfig = { + name: path, + validate: (value: string) => { + try { + yupSchema.validateSync(value); + return undefined; + } catch (err) { + if (err instanceof yup.ValidationError) { + return err.errors.join(", "); + } + throw err; + } + }, + }; + const [field, meta, helpers] = useField(fieldConfig); + const hasError = !!meta.error && meta.touched; + + return ( + + {props.type === "text" && } + {props.type === "array" && ( + + )} + {props.type === "enum" && ( + + )} + {hasError && ( + + + + )} + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss new file mode 100644 index 000000000000..6bedeb454d3a --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.module.scss @@ -0,0 +1,104 @@ +@use "scss/variables"; +@use "scss/colors"; +@use "scss/mixins"; + +.container { + padding: variables.$spacing-sm; + display: flex; + flex-direction: column; + align-items: center; + position: relative; +} + +.connectorImg { + @include mixins.shadow; + + width: 90px; + border-radius: 25px; + padding: variables.$spacing-xl; + margin-top: 55px; +} + +.connectorName { + margin-top: variables.$spacing-md; + min-height: 28px; + width: 100%; + display: flex; + align-items: center; + overflow-x: auto; +} + +.connectorNameText { + margin-left: auto; + margin-right: auto; +} + +.streamsHeader { + display: flex; + align-items: center; + width: 100%; + padding: 0 variables.$spacing-sm 0 variables.$spacing-md; + margin-top: variables.$spacing-2xl; +} + +.streamsHeading { + color: colors.$blue; + flex-grow: 1; +} + +.downloadButton { + width: 100%; +} + +.resetButton { + margin-top: variables.$spacing-sm; +} + +.viewButton { + border: none; + width: 100%; + height: 34px; + flex-shrink: 0; + cursor: pointer; + border-radius: variables.$border-radius-md; + padding: variables.$spacing-md; + display: flex; + align-items: center; + gap: variables.$spacing-sm; + overflow: hidden; +} + +.unselectedViewButton { + background-color: transparent; + color: colors.$dark-blue; + + &:hover { + background-color: colors.$grey-50; + } +} + +.selectedViewButton { + background-color: colors.$dark-blue; + color: colors.$white; +} + +.globalConfigButton { + margin-top: variables.$spacing-xl; +} + +.streamList { + flex-grow: 1; + width: 100%; + margin: variables.$spacing-md 0; + display: flex; + flex-direction: column; + overflow-y: auto; +} + +.streamViewText { + color: inherit; +} + +.emptyStreamViewText { + color: colors.$grey-300; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx new file mode 100644 index 000000000000..2a2b40827cb0 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/BuilderSidebar.tsx @@ -0,0 +1,132 @@ +import { faSliders } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import classnames from "classnames"; +import { useFormikContext } from "formik"; +import { FormattedMessage, useIntl } from "react-intl"; + +import { Button } from "components/ui/Button"; +import { Heading } from "components/ui/Heading"; +import { Text } from "components/ui/Text"; + +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; +import { + BuilderView, + DEFAULT_BUILDER_FORM_VALUES, + useConnectorBuilderState, +} from "services/connectorBuilder/ConnectorBuilderStateService"; + +import { DownloadYamlButton } from "../DownloadYamlButton"; +import { BuilderFormValues } from "../types"; +import { AddStreamButton } from "./AddStreamButton"; +import styles from "./BuilderSidebar.module.scss"; +import { UiYamlToggleButton } from "./UiYamlToggleButton"; + +interface ViewSelectButtonProps { + className?: string; + selected: boolean; + onClick: () => void; +} + +const ViewSelectButton: React.FC> = ({ + children, + className, + selected, + onClick, +}) => { + return ( + + ); +}; + +interface BuilderSidebarProps { + className?: string; + toggleYamlEditor: () => void; +} + +export const BuilderSidebar: React.FC = ({ className, toggleYamlEditor }) => { + const { formatMessage } = useIntl(); + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); + const { yamlManifest, selectedView, setSelectedView, setTestStreamIndex } = useConnectorBuilderState(); + const { values, setValues } = useFormikContext(); + const handleResetForm = () => { + openConfirmationModal({ + text: "connectorBuilder.resetModal.text", + title: "connectorBuilder.resetModal.title", + submitButtonText: "connectorBuilder.resetModal.submitButton", + onSubmit: () => { + setValues(DEFAULT_BUILDER_FORM_VALUES); + setSelectedView("global"); + closeConfirmationModal(); + }, + }); + }; + const handleViewSelect = (selectedView: BuilderView) => { + setSelectedView(selectedView); + if (selectedView !== "global") { + setTestStreamIndex(selectedView); + } + }; + + return ( +
+ + + {/* TODO: replace with uploaded img when that functionality is added */} + {formatMessage({ + +
+ + {values.global?.connectorName} + +
+ + handleViewSelect("global")} + > + + + + +
+ + + + + handleViewSelect(addedStreamNum)} /> +
+ +
+ {values.streams.map(({ name }, num) => ( + handleViewSelect(num)}> + {name && name.trim() ? ( + {name} + ) : ( + + + + )} + + ))} +
+ + + +
+ ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx new file mode 100644 index 000000000000..1d1a40a94518 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/GlobalConfigView.tsx @@ -0,0 +1,21 @@ +import { BuilderCard } from "./BuilderCard"; +import { BuilderField } from "./BuilderField"; + +export const GlobalConfigView: React.FC = () => { + return ( + <> + {/* Not using intl for the labels and tooltips in this component in order to keep maintainence simple */} + + + + + + + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx new file mode 100644 index 000000000000..a59b23b3b533 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/StreamConfigView.tsx @@ -0,0 +1,36 @@ +import { BuilderCard } from "./BuilderCard"; +import { BuilderField } from "./BuilderField"; + +interface StreamConfigViewProps { + streamNum: number; +} + +export const StreamConfigView: React.FC = ({ streamNum }) => { + const streamPath = (path: string) => `streams[${streamNum}].${path}`; + + return ( + + {/* Not using intl for the labels and tooltips in this component in order to keep maintainence simple */} + + + + + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss new file mode 100644 index 000000000000..53e26ff34d47 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.module.scss @@ -0,0 +1,38 @@ +@use "scss/variables"; +@use "scss/colors"; + +.button { + cursor: pointer; + border: variables.$border-thin solid colors.$dark-blue; + background-color: colors.$dark-blue; + border-radius: variables.$border-radius-sm; + padding: 0; + overflow: hidden; + + // absolute positioning so it is in the same spot in both the ui and yaml views + position: absolute; + top: 15px; + left: 51px; + display: grid; + grid-template: 1fr / 1fr 1fr; +} + +.text { + height: 100%; + flex: 1; + display: flex; + align-items: center; + justify-content: center; + font-weight: 700; + padding: 4px 8px; +} + +.selected { + background-color: transparent; + color: colors.$white; +} + +.unselected { + background-color: colors.$white; + color: colors.$dark-blue; +} diff --git a/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx new file mode 100644 index 000000000000..e92f1cb25126 --- /dev/null +++ b/airbyte-webapp/src/components/connectorBuilder/Builder/UiYamlToggleButton.tsx @@ -0,0 +1,39 @@ +import classnames from "classnames"; +import { FormattedMessage } from "react-intl"; + +import { Text } from "components/ui/Text"; + +import styles from "./UiYamlToggleButton.module.scss"; + +interface UiYamlToggleButtonProps { + className?: string; + yamlSelected: boolean; + onClick: () => void; +} + +export const UiYamlToggleButton: React.FC = ({ className, yamlSelected, onClick }) => { + return ( + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx b/airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx similarity index 79% rename from airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx rename to airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx index f5a49a5fc99e..a807239e778f 100644 --- a/airbyte-webapp/src/components/connectorBuilder/YamlEditor/DownloadYamlButton.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/DownloadYamlButton.tsx @@ -22,7 +22,7 @@ export const DownloadYamlButton: React.FC = ({ classNam const downloadButton = ( ); - return yamlIsValid ? ( - downloadButton - ) : ( - - - + return ( +
+ {yamlIsValid ? ( + downloadButton + ) : ( + + + + )} +
); }; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.module.scss index 9f05ee6dc40d..5964f623492d 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.module.scss @@ -35,16 +35,3 @@ max-height: 100%; overflow-y: auto; } - -// add a fade at the bottom of the tabPanel -.tabPanel::after { - content: ""; - position: absolute; - z-index: 1; - bottom: 0; - left: 0; - pointer-events: none; - background-image: linear-gradient(to bottom, colors.$transparentColor, colors.$white 100%); - width: 100%; - height: variables.$spacing-xl; -} diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss index 7f297b2eb5cf..14242399efdc 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.module.scss @@ -1,9 +1,7 @@ @use "scss/variables"; @use "scss/colors"; -.centered { - margin-left: auto; - margin-right: auto; +.container { width: 75%; max-width: 320px; } @@ -11,14 +9,17 @@ .button { padding: variables.$spacing-md; background-color: transparent; - border-radius: variables.$border-radius-lg; + border-radius: variables.$border-radius-md; border: none; display: flex; justify-content: center; + gap: variables.$spacing-md; +} + +.label { + overflow: auto; } .arrow { - margin-left: variables.$spacing-md; - margin-top: -1px; color: colors.$blue; } diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx index 300dd13c1f01..bfad00ad7418 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamSelector.tsx @@ -2,25 +2,23 @@ import { faSortDown } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import capitalize from "lodash/capitalize"; +import { useIntl } from "react-intl"; import { Heading } from "components/ui/Heading"; import { ListBox, ListBoxControlButtonProps } from "components/ui/ListBox"; -import { StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; import styles from "./StreamSelector.module.scss"; interface StreamSelectorProps { className?: string; - streams: StreamsListReadStreamsItem[]; - selectedStream: StreamsListReadStreamsItem; } const ControlButton: React.FC> = ({ selectedOption }) => { return ( <> - + {selectedOption.label} @@ -28,18 +26,32 @@ const ControlButton: React.FC> = ({ selectedOp ); }; -export const StreamSelector: React.FC = ({ className, streams, selectedStream }) => { - const { setSelectedStream } = useConnectorBuilderState(); +export const StreamSelector: React.FC = ({ className }) => { + const { formatMessage } = useIntl(); + const { streams, selectedView, testStreamIndex, setSelectedView, setTestStreamIndex } = useConnectorBuilderState(); const options = streams.map((stream) => { - return { label: capitalize(stream.name), value: stream.name }; + const label = + stream.name && stream.name.trim() ? capitalize(stream.name) : formatMessage({ id: "connectorBuilder.emptyName" }); + return { label, value: stream.name }; }); + const handleStreamSelect = (selectedStreamName: string) => { + const selectedStreamIndex = streams.findIndex((stream) => selectedStreamName === stream.name); + if (selectedStreamIndex >= 0) { + setTestStreamIndex(selectedStreamIndex); + + if (selectedView !== "global" && selectedStreamIndex >= 0) { + setSelectedView(selectedStreamIndex); + } + } + }; + return ( diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss index 0d538a3b55b8..c4ad3c80734c 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.module.scss @@ -8,6 +8,7 @@ align-items: center; gap: variables.$spacing-lg; min-height: 0; + width: 100%; } .resizablePanelsContainer { @@ -18,10 +19,6 @@ z-index: 0; } -.testButton { - width: 100%; -} - .testButtonTooltipContainer { width: 100%; } diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx index 3cb1cfa7e4a1..2bb0459b3a96 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx @@ -10,7 +10,6 @@ import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; import { Tooltip } from "components/ui/Tooltip"; -import { StreamsListReadStreamsItem } from "core/request/ConnectorBuilderClient"; import { useReadStream } from "services/connectorBuilder/ConnectorBuilderApiService"; import { useConnectorBuilderState } from "services/connectorBuilder/ConnectorBuilderStateService"; @@ -18,13 +17,9 @@ import { LogsDisplay } from "./LogsDisplay"; import { ResultDisplay } from "./ResultDisplay"; import styles from "./StreamTester.module.scss"; -interface StreamTesterProps { - selectedStream: StreamsListReadStreamsItem; -} - -export const StreamTester: React.FC = ({ selectedStream }) => { +export const StreamTester: React.FC = () => { const { formatMessage } = useIntl(); - const { jsonManifest, configJson, yamlIsValid } = useConnectorBuilderState(); + const { jsonManifest, configJson, yamlIsValid, streams, testStreamIndex } = useConnectorBuilderState(); const { data: streamReadData, refetch: readStream, @@ -33,7 +28,7 @@ export const StreamTester: React.FC = ({ selectedStream }) => isFetching, } = useReadStream({ manifest: jsonManifest, - stream: selectedStream.name, + stream: streams[testStreamIndex]?.name, config: configJson, }); @@ -60,7 +55,7 @@ export const StreamTester: React.FC = ({ selectedStream }) => const testButton = ( - - - ) : ( - - )} - - ); -}; - -export default ConfirmationControl; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Property/Control.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Property/Control.tsx index 2208ee16b979..a6b9c1d1c396 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Property/Control.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Property/Control.tsx @@ -5,7 +5,6 @@ import { DatePicker } from "components/ui/DatePicker"; import { DropDown } from "components/ui/DropDown"; import { Input } from "components/ui/Input"; import { Multiselect } from "components/ui/Multiselect"; -import { SecretTextArea } from "components/ui/SecretTextArea"; import { TagInput } from "components/ui/TagInput/TagInput"; import { TextArea } from "components/ui/TextArea"; @@ -13,27 +12,16 @@ import { FormBaseItem } from "core/form/types"; import { useExperiment } from "hooks/services/Experiment"; import { isDefined } from "utils/common"; -import ConfirmationControl from "./ConfirmationControl"; +import SecretConfirmationControl from "./SecretConfirmationControl"; interface ControlProps { property: FormBaseItem; name: string; - unfinishedFlows: Record; - addUnfinishedFlow: (key: string, info?: Record) => void; - removeUnfinishedFlow: (key: string) => void; disabled?: boolean; error?: boolean; } -export const Control: React.FC = ({ - property, - name, - addUnfinishedFlow, - removeUnfinishedFlow, - unfinishedFlows, - disabled, - error, -}) => { +export const Control: React.FC = ({ property, name, disabled, error }) => { const [field, meta, helpers] = useField(name); const useDatepickerExperiment = useExperiment("connector.form.useDatepicker", true); @@ -107,46 +95,14 @@ export const Control: React.FC = ({ } else if (property.multiline && !property.isSecret) { return