diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 31c248e30927..cc72a6ea53e4 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2789,6 +2789,8 @@ components: format: uuid disable_cache: type: boolean + notifySchemaChange: + type: boolean SourceUpdate: type: object required: diff --git a/airbyte-bootloader/build.gradle b/airbyte-bootloader/build.gradle index ad9c57c65dbe..d61eaedd6be1 100644 --- a/airbyte-bootloader/build.gradle +++ b/airbyte-bootloader/build.gradle @@ -23,7 +23,7 @@ dependencies { implementation project(':airbyte-config:config-persistence') implementation project(':airbyte-db:db-lib') implementation project(":airbyte-json-validation") - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-persistence:job-persistence') testAnnotationProcessor platform(libs.micronaut.bom) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index bd0481043c1e..a1356c5c8cef 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.28.0 +current_version = 0.28.1 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 24f98960c6db..05181ae86d27 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.28.1 +Low-Code CDK: fix signature _parse_records_and_emit_request_and_responses + ## 0.28.0 Low-Code: improve day_delta macro and MinMaxDatetime component diff --git a/airbyte-cdk/python/README.md b/airbyte-cdk/python/README.md index cbd1a6a75f26..cd74bb8c9c0a 100644 --- a/airbyte-cdk/python/README.md +++ b/airbyte-cdk/python/README.md @@ -71,7 +71,8 @@ If the iteration you are working on includes changes to the models, you might wa ```commandline SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace ``` -This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateProtocolClassFiles` or `./gradlew generateComponentManifestClassFiles`. +This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on +pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateComponentManifestClassFiles`. #### Testing diff --git a/airbyte-cdk/python/airbyte_cdk/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/models/__init__.py index bb6ce3e4a805..e3127e1d519a 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/models/__init__.py @@ -1,3 +1,9 @@ -# generated by generate-protocol-files +# The earlier versions of airbyte-cdk (0.28.0<=) had the airbyte_protocol python classes +# declared inline in the airbyte-cdk code. However, somewhere around Feb 2023 the +# Airbyte Protocol moved to its own repo/PyPi package, called airbyte-protocol-models. +# This directory including the airbyte_protocol.py and well_known_types.py files +# are just wrappers on top of that stand-alone package which do some namespacing magic +# to make the airbyte_protocol python classes available to the airbyte-cdk consumer as part +# of airbyte-cdk rather than a standalone package. from .airbyte_protocol import * from .well_known_types import * diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 1b8339b22700..e634c18acd69 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -2,398 +2,4 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -# generated by datamodel-codegen: -# filename: airbyte_protocol.yaml - -from __future__ import annotations - -from enum import Enum -from typing import Any, Dict, List, Optional, Union - -from pydantic import AnyUrl, BaseModel, Extra, Field - - -class Type(Enum): - RECORD = "RECORD" - STATE = "STATE" - LOG = "LOG" - SPEC = "SPEC" - CONNECTION_STATUS = "CONNECTION_STATUS" - CATALOG = "CATALOG" - TRACE = "TRACE" - CONTROL = "CONTROL" - - -class AirbyteRecordMessage(BaseModel): - class Config: - extra = Extra.allow - - namespace: Optional[str] = Field(None, description="namespace the data is associated with") - stream: str = Field(..., description="stream the data is associated with") - data: Dict[str, Any] = Field(..., description="record data") - emitted_at: int = Field( - ..., - description="when the data was emitted from the source. epoch in millisecond.", - ) - - -class AirbyteStateType(Enum): - GLOBAL = "GLOBAL" - STREAM = "STREAM" - LEGACY = "LEGACY" - - -class StreamDescriptor(BaseModel): - class Config: - extra = Extra.allow - - name: str - namespace: Optional[str] = None - - -class AirbyteStateBlob(BaseModel): - pass - - class Config: - extra = Extra.allow - - -class Level(Enum): - FATAL = "FATAL" - ERROR = "ERROR" - WARN = "WARN" - INFO = "INFO" - DEBUG = "DEBUG" - TRACE = "TRACE" - - -class AirbyteLogMessage(BaseModel): - class Config: - extra = Extra.allow - - level: Level = Field(..., description="log level") - message: str = Field(..., description="log message") - stack_trace: Optional[str] = Field( - None, - description="an optional stack trace if the log message corresponds to an exception", - ) - - -class TraceType(Enum): - ERROR = "ERROR" - ESTIMATE = "ESTIMATE" - - -class FailureType(Enum): - system_error = "system_error" - config_error = "config_error" - - -class AirbyteErrorTraceMessage(BaseModel): - class Config: - extra = Extra.allow - - message: str = Field(..., description="A user-friendly message that indicates the cause of the error") - internal_message: Optional[str] = Field(None, description="The internal error that caused the failure") - stack_trace: Optional[str] = Field(None, description="The full stack trace of the error") - failure_type: Optional[FailureType] = Field(None, description="The type of error") - - -class EstimateType(Enum): - STREAM = "STREAM" - SYNC = "SYNC" - - -class AirbyteEstimateTraceMessage(BaseModel): - class Config: - extra = Extra.allow - - name: str = Field(..., description="The name of the stream") - type: EstimateType = Field( - ..., - description="Estimates are either per-stream (STREAM) or for the entire sync (SYNC). STREAM is preferred, and requires the source to count how many records are about to be emitted per-stream (e.g. there will be 100 rows from this table emitted). For the rare source which cannot tell which stream a record belongs to before reading (e.g. CDC databases), SYNC estimates can be emitted. Sources should not emit both STREAM and SOURCE estimates within a sync.\n", - title="estimate type", - ) - namespace: Optional[str] = Field(None, description="The namespace of the stream") - row_estimate: Optional[int] = Field( - None, - description="The estimated number of rows to be emitted by this sync for this stream", - ) - byte_estimate: Optional[int] = Field( - None, - description="The estimated number of bytes to be emitted by this sync for this stream", - ) - - -class OrchestratorType(Enum): - CONNECTOR_CONFIG = "CONNECTOR_CONFIG" - - -class AirbyteControlConnectorConfigMessage(BaseModel): - class Config: - extra = Extra.allow - - config: Dict[str, Any] = Field(..., description="the config items from this connector's spec to update") - - -class Status(Enum): - SUCCEEDED = "SUCCEEDED" - FAILED = "FAILED" - - -class AirbyteConnectionStatus(BaseModel): - class Config: - extra = Extra.allow - - status: Status - message: Optional[str] = None - - -class SyncMode(Enum): - full_refresh = "full_refresh" - incremental = "incremental" - - -class DestinationSyncMode(Enum): - append = "append" - overwrite = "overwrite" - append_dedup = "append_dedup" - - -class OAuth2Specification(BaseModel): - class Config: - extra = Extra.allow - - rootObject: Optional[List[Union[str, int]]] = Field( - None, - description="A list of strings representing a pointer to the root object which contains any oauth parameters in the ConnectorSpecification.\nExamples:\nif oauth parameters were contained inside the top level, rootObject=[] If they were nested inside another object {'credentials': {'app_id' etc...}, rootObject=['credentials'] If they were inside a oneOf {'switch': {oneOf: [{client_id...}, {non_oauth_param]}}, rootObject=['switch', 0] ", - ) - oauthFlowInitParameters: Optional[List[List[str]]] = Field( - None, - description="Pointers to the fields in the rootObject needed to obtain the initial refresh/access tokens for the OAuth flow. Each inner array represents the path in the rootObject of the referenced field. For example. Assume the rootObject contains params 'app_secret', 'app_id' which are needed to get the initial refresh token. If they are not nested in the rootObject, then the array would look like this [['app_secret'], ['app_id']] If they are nested inside an object called 'auth_params' then this array would be [['auth_params', 'app_secret'], ['auth_params', 'app_id']]", - ) - oauthFlowOutputParameters: Optional[List[List[str]]] = Field( - None, - description="Pointers to the fields in the rootObject which can be populated from successfully completing the oauth flow using the init parameters. This is typically a refresh/access token. Each inner array represents the path in the rootObject of the referenced field.", - ) - - -class AuthType(Enum): - oauth2_0 = "oauth2.0" - - -class AuthSpecification(BaseModel): - auth_type: Optional[AuthType] = None - oauth2Specification: Optional[OAuth2Specification] = Field( - None, - description="If the connector supports OAuth, this field should be non-null.", - ) - - -class AuthFlowType(Enum): - oauth2_0 = "oauth2.0" - oauth1_0 = "oauth1.0" - - -class OAuthConfigSpecification(BaseModel): - class Config: - extra = Extra.allow - - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\n\nExamples:\n\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - ) - complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", - ) - complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\n\nExamples:\n\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - ) - complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\n\nExamples:\n\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", - ) - - -class AirbyteStreamState(BaseModel): - class Config: - extra = Extra.allow - - stream_descriptor: StreamDescriptor - stream_state: Optional[AirbyteStateBlob] = None - - -class AirbyteGlobalState(BaseModel): - class Config: - extra = Extra.allow - - shared_state: Optional[AirbyteStateBlob] = None - stream_states: List[AirbyteStreamState] - - -class AirbyteTraceMessage(BaseModel): - class Config: - extra = Extra.allow - - type: TraceType = Field(..., description="the type of trace message", title="trace type") - emitted_at: float = Field(..., description="the time in ms that the message was emitted") - error: Optional[AirbyteErrorTraceMessage] = Field(None, description="error trace message: the error object") - estimate: Optional[AirbyteEstimateTraceMessage] = Field( - None, - description="Estimate trace message: a guess at how much data will be produced in this sync", - ) - - -class AirbyteControlMessage(BaseModel): - class Config: - extra = Extra.allow - - type: OrchestratorType = Field(..., description="the type of orchestrator message", title="orchestrator type") - emitted_at: float = Field(..., description="the time in ms that the message was emitted") - connectorConfig: Optional[AirbyteControlConnectorConfigMessage] = Field( - None, - description="connector config orchestrator message: the updated config for the platform to store for this connector", - ) - - -class AirbyteStream(BaseModel): - class Config: - extra = Extra.allow - - name: str = Field(..., description="Stream's name.") - json_schema: Dict[str, Any] = Field(..., description="Stream schema using Json Schema specs.") - supported_sync_modes: List[SyncMode] = Field(..., description="List of sync modes supported by this stream.", min_items=1) - source_defined_cursor: Optional[bool] = Field( - None, - description="If the source defines the cursor field, then any other cursor field inputs will be ignored. If it does not, either the user_provided one is used, or the default one is used as a backup.", - ) - default_cursor_field: Optional[List[str]] = Field( - None, - description="Path to the field that will be used to determine if a record is new or modified since the last sync. If not provided by the source, the end user will have to specify the comparable themselves.", - ) - source_defined_primary_key: Optional[List[List[str]]] = Field( - None, - description="If the source defines the primary key, paths to the fields that will be used as a primary key. If not provided by the source, the end user will have to specify the primary key themselves.", - ) - namespace: Optional[str] = Field( - None, - description="Optional Source-defined namespace. Currently only used by JDBC destinations to determine what schema to write to. Airbyte streams from the same sources should have the same namespace.", - ) - - -class ConfiguredAirbyteStream(BaseModel): - class Config: - extra = Extra.allow - - stream: AirbyteStream - sync_mode: SyncMode - cursor_field: Optional[List[str]] = Field( - None, - description="Path to the field that will be used to determine if a record is new or modified since the last sync. This field is REQUIRED if `sync_mode` is `incremental`. Otherwise it is ignored.", - ) - destination_sync_mode: DestinationSyncMode - primary_key: Optional[List[List[str]]] = Field( - None, - description="Paths to the fields that will be used as primary key. This field is REQUIRED if `destination_sync_mode` is `*_dedup`. Otherwise it is ignored.", - ) - - -class AdvancedAuth(BaseModel): - auth_flow_type: Optional[AuthFlowType] = None - predicate_key: Optional[List[str]] = Field( - None, - description="Json Path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", - ) - predicate_value: Optional[str] = Field( - None, - description="Value of the predicate_key fields for the advanced auth to be applicable.", - ) - oauth_config_specification: Optional[OAuthConfigSpecification] = None - - -class ConnectorSpecification(BaseModel): - class Config: - extra = Extra.allow - - documentationUrl: Optional[AnyUrl] = None - changelogUrl: Optional[AnyUrl] = None - connectionSpecification: Dict[str, Any] = Field( - ..., - description="ConnectorDefinition specific blob. Must be a valid JSON string.", - ) - supportsIncremental: Optional[bool] = Field( - None, - description="(deprecated) If the connector supports incremental mode or not.", - ) - supportsNormalization: Optional[bool] = Field(False, description="If the connector supports normalization or not.") - supportsDBT: Optional[bool] = Field(False, description="If the connector supports DBT or not.") - supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field( - None, description="List of destination sync modes supported by the connector" - ) - authSpecification: Optional[AuthSpecification] = Field(None, description="deprecated, switching to advanced_auth instead") - advanced_auth: Optional[AdvancedAuth] = Field( - None, - description="Additional and optional specification object to describe what an 'advanced' Auth flow would need to function.\n - A connector should be able to fully function with the configuration as described by the ConnectorSpecification in a 'basic' mode.\n - The 'advanced' mode provides easier UX for the user with UI improvements and automations. However, this requires further setup on the\n server side by instance or workspace admins beforehand. The trade-off is that the user does not have to provide as many technical\n inputs anymore and the auth process is faster and easier to complete.", - ) - protocol_version: Optional[str] = Field( - None, - description="the Airbyte Protocol version supported by the connector. Protocol versioning uses SemVer. ", - ) - - -class AirbyteStateMessage(BaseModel): - class Config: - extra = Extra.allow - - type: Optional[AirbyteStateType] = None - stream: Optional[AirbyteStreamState] = None - global_: Optional[AirbyteGlobalState] = Field(None, alias="global") - data: Optional[Dict[str, Any]] = Field(None, description="(Deprecated) the state data") - - -class AirbyteCatalog(BaseModel): - class Config: - extra = Extra.allow - - streams: List[AirbyteStream] - - -class ConfiguredAirbyteCatalog(BaseModel): - class Config: - extra = Extra.allow - - streams: List[ConfiguredAirbyteStream] - - -class AirbyteMessage(BaseModel): - class Config: - extra = Extra.allow - - type: Type = Field(..., description="Message type") - log: Optional[AirbyteLogMessage] = Field( - None, - description="log message: any kind of logging you want the platform to know about.", - ) - spec: Optional[ConnectorSpecification] = None - connectionStatus: Optional[AirbyteConnectionStatus] = None - catalog: Optional[AirbyteCatalog] = Field(None, description="catalog message: the catalog") - record: Optional[AirbyteRecordMessage] = Field(None, description="record message: the record") - state: Optional[AirbyteStateMessage] = Field( - None, - description="schema message: the state. Must be the last message produced. The platform uses this information", - ) - trace: Optional[AirbyteTraceMessage] = Field( - None, - description="trace message: a message to communicate information about the status and performance of a connector", - ) - control: Optional[AirbyteControlMessage] = Field( - None, - description="connector config message: a message to communicate an updated configuration from a connector that should be persisted", - ) - - -class AirbyteProtocol(BaseModel): - airbyte_message: Optional[AirbyteMessage] = None - configured_airbyte_catalog: Optional[ConfiguredAirbyteCatalog] = None +from airbyte_protocol.models.airbyte_protocol import * diff --git a/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py b/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py index 4ea48602c743..0cc409c7e070 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py +++ b/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py @@ -2,85 +2,4 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -# generated by datamodel-codegen: -# filename: well_known_types.yaml - -from __future__ import annotations - -from enum import Enum -from typing import Any, Union - -from pydantic import BaseModel, Field, constr - - -class Model(BaseModel): - __root__: Any - - -class String(BaseModel): - __root__: str = Field(..., description="Arbitrary text") - - -class BinaryData(BaseModel): - __root__: constr(regex=r"^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$") = Field( - ..., - description="Arbitrary binary data. Represented as base64-encoded strings in the JSON transport. In the future, if we support other transports, may be encoded differently.\n", - ) - - -class Date(BaseModel): - __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}( BC)?$") = Field( - ..., description="RFC 3339§5.6's full-date format, extended with BC era support" - ) - - -class TimestampWithTimezone(BaseModel): - __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+\-]\d{1,2}:\d{2})( BC)?$") = Field( - ..., - description='An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339§5.6\'s date-time format, requiring a "T" separator, and extended with BC era support. Note that we do _not_ accept Unix epochs here.\n', - ) - - -class TimestampWithoutTimezone(BaseModel): - __root__: constr(regex=r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?( BC)?$") = Field( - ..., - description='Also known as a localdatetime, or just datetime. Under RFC 3339§5.6, this would be represented as `full-date "T" partial-time`, extended with BC era support.\n', - ) - - -class TimeWithTimezone(BaseModel): - __root__: constr(regex=r"^\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+\-]\d{1,2}:\d{2})$") = Field(..., description="An RFC 3339§5.6 full-time") - - -class TimeWithoutTimezone(BaseModel): - __root__: constr(regex=r"^\d{2}:\d{2}:\d{2}(\.\d+)?$") = Field(..., description="An RFC 3339§5.6 partial-time") - - -class NumberEnum(Enum): - Infinity = "Infinity" - _Infinity = "-Infinity" - NaN = "NaN" - - -class Number(BaseModel): - __root__: Union[Any, NumberEnum] = Field( - ..., - description="Note the mix of regex validation for normal numbers, and enum validation for special values.", - ) - - -class IntegerEnum(Enum): - Infinity = "Infinity" - _Infinity = "-Infinity" - NaN = "NaN" - - -class Integer(BaseModel): - __root__: Union[Any, IntegerEnum] - - -class Boolean(BaseModel): - __root__: bool = Field( - ..., - description="Note the direct usage of a primitive boolean rather than string. Unlike Numbers and Integers, we don't expect unusual values here.", - ) +from airbyte_protocol.models.well_known_types import * diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index aaed68c60173..a9b06deca222 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -407,7 +407,7 @@ def state(self, value: StreamState): """State setter, accept state serialized by state getter.""" self.stream_slicer.update_cursor(value) - def _parse_records_and_emit_request_and_responses(self, request, response, stream_slice, stream_state) -> Iterable[StreamData]: + def _parse_records_and_emit_request_and_responses(self, request, response, stream_state, stream_slice) -> Iterable[StreamData]: # Only emit requests and responses when running in debug mode if self.logger.isEnabledFor(logging.DEBUG): yield _prepared_request_to_airbyte_message(request) diff --git a/airbyte-cdk/python/bin/generate-protocol-files.sh b/airbyte-cdk/python/bin/generate-protocol-files.sh deleted file mode 100755 index a758093c60b9..000000000000 --- a/airbyte-cdk/python/bin/generate-protocol-files.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e - -[ -z "$ROOT_DIR" ] && exit 1 - -YAML_DIR=airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol -OUTPUT_DIR=airbyte-cdk/python/airbyte_cdk/models - -function main() { - rm -rf "$ROOT_DIR/$OUTPUT_DIR"/*.py - echo "# generated by generate-protocol-files" > "$ROOT_DIR/$OUTPUT_DIR"/__init__.py - - for f in "$ROOT_DIR/$YAML_DIR"/*.yaml; do - filename_wo_ext=$(basename "$f" | cut -d . -f 1) - echo "from .$filename_wo_ext import *" >> "$ROOT_DIR/$OUTPUT_DIR"/__init__.py - - docker run --user "$(id -u):$(id -g)" -v "$ROOT_DIR":/airbyte airbyte/code-generator:dev \ - --input "/airbyte/$YAML_DIR/$filename_wo_ext.yaml" \ - --output "/airbyte/$OUTPUT_DIR/$filename_wo_ext.py" \ - --use-title-as-name \ - --disable-timestamp - done -} - -main "$@" diff --git a/airbyte-cdk/python/build.gradle b/airbyte-cdk/python/build.gradle index addc32e7c80f..2784943b22de 100644 --- a/airbyte-cdk/python/build.gradle +++ b/airbyte-cdk/python/build.gradle @@ -7,12 +7,6 @@ airbytePython { moduleDirectory 'airbyte_cdk' } -task generateProtocolClassFiles(type: Exec) { - environment 'ROOT_DIR', rootDir.absolutePath - commandLine 'bin/generate-protocol-files.sh' - dependsOn ':tools:code-generator:airbyteDocker' -} - task generateComponentManifestClassFiles(type: Exec) { environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/generate-component-manifest-files.sh' @@ -24,11 +18,6 @@ task validateSourceYamlManifest(type: Exec) { commandLine 'bin/validate-yaml-schema.sh' } -blackFormat.dependsOn generateProtocolClassFiles -isortFormat.dependsOn generateProtocolClassFiles -flakeCheck.dependsOn generateProtocolClassFiles -installReqs.dependsOn generateProtocolClassFiles - blackFormat.dependsOn generateComponentManifestClassFiles isortFormat.dependsOn generateComponentManifestClassFiles flakeCheck.dependsOn generateComponentManifestClassFiles diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 39cb909197db..1be2b946e561 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.28.0", + version="0.28.1", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -44,6 +44,7 @@ packages=find_packages(exclude=("unit_tests",)), package_data={"airbyte_cdk": ["py.typed", "sources/declarative/declarative_component_schema.yaml"]}, install_requires=[ + "airbyte-protocol-models==1.0.0", "backoff", # pinned to the last working version for us temporarily while we fix "dataclasses-jsonschema==2.15.1", diff --git a/airbyte-commons-protocol/build.gradle b/airbyte-commons-protocol/build.gradle index cd5f7132b5fb..5787c7af290d 100644 --- a/airbyte-commons-protocol/build.gradle +++ b/airbyte-commons-protocol/build.gradle @@ -5,7 +5,7 @@ dependencies { implementation libs.bundles.micronaut.annotation testImplementation libs.bundles.micronaut.test - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-json-validation') } diff --git a/airbyte-commons-server/build.gradle b/airbyte-commons-server/build.gradle index 5932142dadfe..ec25c650ae7a 100644 --- a/airbyte-commons-server/build.gradle +++ b/airbyte-commons-server/build.gradle @@ -39,7 +39,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(":airbyte-json-validation") implementation project(':airbyte-oauth') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-persistence:job-persistence') implementation 'com.github.slugify:slugify:2.4' diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java index 8da81b533528..f9e479df97bd 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java @@ -70,6 +70,7 @@ import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.WebUrlHelper; import io.airbyte.persistence.job.models.Job; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; @@ -81,8 +82,10 @@ import java.util.Optional; import java.util.UUID; import javax.validation.constraints.NotNull; +import lombok.extern.slf4j.Slf4j; @Singleton +@Slf4j public class SchedulerHandler { private static final HashFunction HASH_FUNCTION = Hashing.md5(); @@ -100,6 +103,7 @@ public class SchedulerHandler { private final JobConverter jobConverter; private final EventRunner eventRunner; private final FeatureFlags envVariableFeatureFlags; + private final WebUrlHelper webUrlHelper; // TODO: Convert to be fully using micronaut public SchedulerHandler(final ConfigRepository configRepository, @@ -111,7 +115,8 @@ public SchedulerHandler(final ConfigRepository configRepository, final LogConfigs logConfigs, final EventRunner eventRunner, final ConnectionsHandler connectionsHandler, - final FeatureFlags envVariableFeatureFlags) { + final FeatureFlags envVariableFeatureFlags, + final WebUrlHelper webUrlHelper) { this( configRepository, secretsRepositoryWriter, @@ -122,7 +127,8 @@ public SchedulerHandler(final ConfigRepository configRepository, eventRunner, new JobConverter(workerEnvironment, logConfigs), connectionsHandler, - envVariableFeatureFlags); + envVariableFeatureFlags, + webUrlHelper); } @VisibleForTesting @@ -135,7 +141,8 @@ public SchedulerHandler(final ConfigRepository configRepository, final EventRunner eventRunner, final JobConverter jobConverter, final ConnectionsHandler connectionsHandler, - final FeatureFlags envVariableFeatureFlags) { + final FeatureFlags envVariableFeatureFlags, + final WebUrlHelper webUrlHelper) { this.configRepository = configRepository; this.secretsRepositoryWriter = secretsRepositoryWriter; this.synchronousSchedulerClient = synchronousSchedulerClient; @@ -146,6 +153,7 @@ public SchedulerHandler(final ConfigRepository configRepository, this.jobConverter = jobConverter; this.connectionsHandler = connectionsHandler; this.envVariableFeatureFlags = envVariableFeatureFlags; + this.webUrlHelper = webUrlHelper; } public CheckConnectionRead checkSourceConnectionFromSourceId(final SourceIdRequestBody sourceIdRequestBody) @@ -272,7 +280,7 @@ public SourceDiscoverSchemaRead discoverSchemaForSourceFromSourceId(final Source if (persistedCatalogId.isSuccess() && discoverSchemaRequestBody.getConnectionId() != null) { // modify discoveredSchema object to add CatalogDiff, containsBreakingChange, and connectionStatus - generateCatalogDiffsAndDisableConnectionsIfNeeded(discoveredSchema, discoverSchemaRequestBody); + generateCatalogDiffsAndDisableConnectionsIfNeeded(discoveredSchema, discoverSchemaRequestBody, source.getWorkspaceId()); } return discoveredSchema; @@ -394,9 +402,10 @@ public JobInfoRead cancelJob(final JobIdRequestBody jobIdRequestBody) throws IOE // determine whether 1. the source schema change resulted in a broken connection or 2. the user // wants the connection disabled when non-breaking changes are detected. If so, disable that // connection. Modify the current discoveredSchema object to add a CatalogDiff, - // containsBreakingChange paramter, and connectionStatus parameter. + // containsBreakingChange parameter, and connectionStatus parameter. private void generateCatalogDiffsAndDisableConnectionsIfNeeded(final SourceDiscoverSchemaRead discoveredSchema, - final SourceDiscoverSchemaRequestBody discoverSchemaRequestBody) + final SourceDiscoverSchemaRequestBody discoverSchemaRequestBody, + final UUID workspaceId) throws JsonValidationException, ConfigNotFoundException, IOException { final ConnectionReadList connectionsForSource = connectionsHandler.listConnectionsForSource(discoverSchemaRequestBody.getSourceId(), false); for (final ConnectionRead connectionRead : connectionsForSource.getConnections()) { @@ -418,12 +427,23 @@ private void generateCatalogDiffsAndDisableConnectionsIfNeeded(final SourceDisco } updateObject.status(connectionStatus); connectionsHandler.updateConnection(updateObject); + if (shouldNotifySchemaChange(diff, connectionRead, discoverSchemaRequestBody)) { + final String url = webUrlHelper.getConnectionUrl(workspaceId, connectionRead.getConnectionId()); + eventRunner.sendSchemaChangeNotification(connectionRead.getConnectionId(), url); + } if (connectionRead.getConnectionId().equals(discoverSchemaRequestBody.getConnectionId())) { discoveredSchema.catalogDiff(diff).breakingChange(containsBreakingChange).connectionStatus(connectionStatus); } } } + private boolean shouldNotifySchemaChange(final CatalogDiff diff, + final ConnectionRead connectionRead, + final SourceDiscoverSchemaRequestBody requestBody) { + return !diff.getTransforms().isEmpty() && connectionRead.getNotifySchemaChanges() && requestBody.getNotifySchemaChange() != null + && requestBody.getNotifySchemaChange(); + } + private boolean shouldDisableConnection(final boolean containsBreakingChange, final NonBreakingChangesPreference preference, final CatalogDiff diff) { diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java index 63b43bbd77fe..b863e56cdedb 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java @@ -413,7 +413,8 @@ private Optional getRefreshedSchema(final UUID sourceI final SourceDiscoverSchemaRequestBody discoverSchemaReadReq = new SourceDiscoverSchemaRequestBody() .sourceId(sourceId) .disableCache(true) - .connectionId(connectionId); + .connectionId(connectionId) + .notifySchemaChange(false); final SourceDiscoverSchemaRead schemaRead = schedulerHandler.discoverSchemaForSourceFromSourceId(discoverSchemaReadReq); return Optional.ofNullable(schemaRead); } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/scheduler/EventRunner.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/scheduler/EventRunner.java index 932202b8fcbd..59ce6f4394f1 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/scheduler/EventRunner.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/scheduler/EventRunner.java @@ -28,4 +28,6 @@ public interface EventRunner { void update(final UUID connectionId); + void sendSchemaChangeNotification(final UUID connectionId, final String url); + } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/scheduler/TemporalEventRunner.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/scheduler/TemporalEventRunner.java index 989f66c8fdea..9e6ed0068e51 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/scheduler/TemporalEventRunner.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/scheduler/TemporalEventRunner.java @@ -54,4 +54,9 @@ public void update(final UUID connectionId) { temporalClient.update(connectionId); } + @Override + public void sendSchemaChangeNotification(final UUID connectionId, final String url) { + temporalClient.sendSchemaChangeNotification(connectionId, url); + } + } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java index a5a292dace0c..50b8f9982042 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java @@ -19,9 +19,11 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import com.google.common.collect.Lists; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.model.generated.CatalogDiff; import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.ConnectionIdRequestBody; @@ -77,6 +79,7 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.WebUrlHelper; import io.airbyte.persistence.job.models.Job; import io.airbyte.persistence.job.models.JobStatus; import io.airbyte.protocol.models.AirbyteCatalog; @@ -113,6 +116,7 @@ class SchedulerHandlerTest { private static final String DOGS = "dogs"; private static final String SHOES = "shoes"; private static final String SKU = "sku"; + private static final String CONNECTION_URL = "connection_url"; private static final AirbyteCatalog airbyteCatalog = CatalogHelpers.createAirbyteCatalog(SHOES, Field.of(SKU, JsonSchemaType.STRING)); @@ -153,6 +157,7 @@ class SchedulerHandlerTest { private JobConverter jobConverter; private ConnectionsHandler connectionsHandler; private EnvVariableFeatureFlags envVariableFeatureFlags; + private WebUrlHelper webUrlHelper; @BeforeEach void setup() { @@ -176,6 +181,7 @@ void setup() { eventRunner = mock(EventRunner.class); connectionsHandler = mock(ConnectionsHandler.class); envVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); + webUrlHelper = mock(WebUrlHelper.class); jobConverter = spy(new JobConverter(WorkerEnvironment.DOCKER, LogConfigs.EMPTY)); @@ -189,7 +195,8 @@ void setup() { eventRunner, jobConverter, connectionsHandler, - envVariableFeatureFlags); + envVariableFeatureFlags, + webUrlHelper); } @Test @@ -561,13 +568,14 @@ void testDiscoverSchemaForSourceFromSourceIdFailed() throws IOException, JsonVal } @Test - void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreaking() throws IOException, JsonValidationException, ConfigNotFoundException { + void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreaking() + throws IOException, JsonValidationException, ConfigNotFoundException, InterruptedException, ApiException { final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); final UUID connectionId = UUID.randomUUID(); final UUID discoveredCatalogId = UUID.randomUUID(); final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; final SourceDiscoverSchemaRequestBody request = - new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true).notifySchemaChange(true); final StreamTransform streamTransform = new StreamTransform().transformType(TransformTypeEnum.REMOVE_STREAM) .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name(DOGS)); final CatalogDiff catalogDiff = new CatalogDiff().addTransformsItem(streamTransform); @@ -582,6 +590,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreaking() throws IOExcept when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION), false)) .thenReturn(discoverResponse); + when(webUrlHelper.getConnectionUrl(source.getWorkspaceId(), connectionId)).thenReturn(CONNECTION_URL); when(discoverResponse.isSuccess()).thenReturn(true); when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); @@ -591,7 +600,8 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreaking() throws IOExcept CatalogHelpers.createAirbyteStream(DOGS, Field.of(NAME, JsonSchemaType.STRING)))); final ConnectionRead connectionRead = - new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).connectionId(connectionId); + new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).connectionId(connectionId) + .notifySchemaChanges(true); when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); final ConnectionReadList connectionReadList = new ConnectionReadList().connections(List.of(connectionRead)); @@ -610,17 +620,18 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreaking() throws IOExcept final SourceDiscoverSchemaRead actual = schedulerHandler.discoverSchemaForSourceFromSourceId(request); assertEquals(actual.getCatalogDiff(), catalogDiff); assertEquals(actual.getCatalog(), expectedActorCatalog); + verify(eventRunner).sendSchemaChangeNotification(connectionId, CONNECTION_URL); } @Test void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionPreferenceNoFeatureFlag() - throws IOException, JsonValidationException, ConfigNotFoundException { + throws IOException, JsonValidationException, ConfigNotFoundException, InterruptedException, ApiException { final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); final UUID connectionId = UUID.randomUUID(); final UUID discoveredCatalogId = UUID.randomUUID(); final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; final SourceDiscoverSchemaRequestBody request = - new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true).notifySchemaChange(true); final StreamTransform streamTransform = new StreamTransform().transformType(TransformTypeEnum.REMOVE_STREAM) .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name(DOGS)); final CatalogDiff catalogDiff = new CatalogDiff().addTransformsItem(streamTransform); @@ -636,6 +647,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION), false)) .thenReturn(discoverResponse); + when(webUrlHelper.getConnectionUrl(source.getWorkspaceId(), connectionId)).thenReturn(CONNECTION_URL); when(discoverResponse.isSuccess()).thenReturn(true); when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); @@ -646,7 +658,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP final ConnectionRead connectionRead = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).nonBreakingChangesPreference( - NonBreakingChangesPreference.DISABLE).status(ConnectionStatus.ACTIVE).connectionId(connectionId); + NonBreakingChangesPreference.DISABLE).status(ConnectionStatus.ACTIVE).connectionId(connectionId).notifySchemaChanges(true); when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); final ConnectionReadList connectionReadList = new ConnectionReadList().connections(List.of(connectionRead)); @@ -666,6 +678,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP assertEquals(actual.getCatalogDiff(), catalogDiff); assertEquals(actual.getCatalog(), expectedActorCatalog); assertEquals(actual.getConnectionStatus(), ConnectionStatus.ACTIVE); + verify(eventRunner).sendSchemaChangeNotification(connectionId, CONNECTION_URL); } @Test @@ -676,7 +689,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP final UUID discoveredCatalogId = UUID.randomUUID(); final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; final SourceDiscoverSchemaRequestBody request = - new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true).notifySchemaChange(true); final StreamTransform streamTransform = new StreamTransform().transformType(TransformTypeEnum.REMOVE_STREAM) .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name(DOGS)); final CatalogDiff catalogDiff = new CatalogDiff().addTransformsItem(streamTransform); @@ -702,7 +715,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP final ConnectionRead connectionRead = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).nonBreakingChangesPreference( - NonBreakingChangesPreference.DISABLE).connectionId(connectionId); + NonBreakingChangesPreference.DISABLE).connectionId(connectionId).notifySchemaChanges(false); when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); final ConnectionReadList connectionReadList = new ConnectionReadList().connections(List.of(connectionRead)); @@ -722,16 +735,18 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP assertEquals(actual.getCatalogDiff(), catalogDiff); assertEquals(actual.getCatalog(), expectedActorCatalog); assertEquals(actual.getConnectionStatus(), ConnectionStatus.INACTIVE); + verifyNoInteractions(eventRunner); } @Test - void testDiscoverSchemaFromSourceIdWithConnectionIdBreaking() throws IOException, JsonValidationException, ConfigNotFoundException { + void testDiscoverSchemaFromSourceIdWithConnectionIdBreaking() + throws IOException, JsonValidationException, ConfigNotFoundException, InterruptedException, ApiException { final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); final UUID connectionId = UUID.randomUUID(); final UUID discoveredCatalogId = UUID.randomUUID(); final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; final SourceDiscoverSchemaRequestBody request = - new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true).notifySchemaChange(true); final StreamTransform streamTransform = new StreamTransform().transformType(TransformTypeEnum.UPDATE_STREAM) .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name(DOGS)).addUpdateStreamItem(new FieldTransform().transformType( FieldTransform.TransformTypeEnum.REMOVE_FIELD).breaking(true)); @@ -747,6 +762,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdBreaking() throws IOException when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION), false)) .thenReturn(discoverResponse); + when(webUrlHelper.getConnectionUrl(source.getWorkspaceId(), connectionId)).thenReturn(CONNECTION_URL); when(discoverResponse.isSuccess()).thenReturn(true); when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); @@ -757,7 +773,8 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdBreaking() throws IOException final ConnectionRead connectionRead = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).status(ConnectionStatus.ACTIVE) - .connectionId(connectionId); + .connectionId(connectionId) + .notifySchemaChanges(true); when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); final ConnectionReadList connectionReadList = new ConnectionReadList().connections(List.of(connectionRead)); @@ -780,16 +797,18 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdBreaking() throws IOException assertEquals(actual.getCatalog(), expectedActorCatalog); assertEquals(actual.getConnectionStatus(), ConnectionStatus.ACTIVE); verify(connectionsHandler).updateConnection(expectedConnectionUpdate); + verify(eventRunner).sendSchemaChangeNotification(connectionId, CONNECTION_URL); } @Test - void testDiscoverSchemaFromSourceIdWithConnectionIdBreakingFeatureFlagOn() throws IOException, JsonValidationException, ConfigNotFoundException { + void testDiscoverSchemaFromSourceIdWithConnectionIdBreakingFeatureFlagOn() + throws IOException, JsonValidationException, ConfigNotFoundException, InterruptedException, ApiException { final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); final UUID connectionId = UUID.randomUUID(); final UUID discoveredCatalogId = UUID.randomUUID(); final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; final SourceDiscoverSchemaRequestBody request = - new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true).notifySchemaChange(true); final StreamTransform streamTransform = new StreamTransform().transformType(TransformTypeEnum.UPDATE_STREAM) .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name(DOGS)).addUpdateStreamItem(new FieldTransform().transformType( FieldTransform.TransformTypeEnum.REMOVE_FIELD).breaking(true)); @@ -806,6 +825,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdBreakingFeatureFlagOn() throw when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION), false)) .thenReturn(discoverResponse); + when(webUrlHelper.getConnectionUrl(source.getWorkspaceId(), connectionId)).thenReturn(CONNECTION_URL); when(discoverResponse.isSuccess()).thenReturn(true); when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); @@ -815,7 +835,8 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdBreakingFeatureFlagOn() throw CatalogHelpers.createAirbyteStream(DOGS, Field.of(NAME, JsonSchemaType.STRING)))); final ConnectionRead connectionRead = - new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).connectionId(connectionId); + new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).connectionId(connectionId) + .notifySchemaChanges(true); when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); final ConnectionReadList connectionReadList = new ConnectionReadList().connections(List.of(connectionRead)); @@ -838,6 +859,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdBreakingFeatureFlagOn() throw assertEquals(actual.getCatalog(), expectedActorCatalog); assertEquals(actual.getConnectionStatus(), ConnectionStatus.INACTIVE); verify(connectionsHandler).updateConnection(expectedConnectionUpdate); + verify(eventRunner).sendSchemaChangeNotification(connectionId, CONNECTION_URL); } @Test @@ -848,7 +870,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP final UUID discoveredCatalogId = UUID.randomUUID(); final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; final SourceDiscoverSchemaRequestBody request = - new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true).notifySchemaChange(true); final CatalogDiff catalogDiff = new CatalogDiff(); final StandardSourceDefinition sourceDef = new StandardSourceDefinition() .withDockerRepository(SOURCE_DOCKER_REPO) @@ -872,7 +894,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP final ConnectionRead connectionRead = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).nonBreakingChangesPreference( - NonBreakingChangesPreference.DISABLE).status(ConnectionStatus.INACTIVE).connectionId(connectionId); + NonBreakingChangesPreference.DISABLE).status(ConnectionStatus.INACTIVE).connectionId(connectionId).notifySchemaChanges(false); when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); final ConnectionReadList connectionReadList = new ConnectionReadList().connections(List.of(connectionRead)); @@ -892,10 +914,13 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionP assertEquals(actual.getCatalogDiff(), catalogDiff); assertEquals(actual.getCatalog(), expectedActorCatalog); assertEquals(actual.getConnectionStatus(), ConnectionStatus.INACTIVE); + // notification preferences are turned on, but there is no schema diff detected + verifyNoInteractions(eventRunner); } @Test - void testDiscoverSchemaForSourceMultipleConnectionsFeatureFlagOn() throws IOException, JsonValidationException, ConfigNotFoundException { + void testDiscoverSchemaForSourceMultipleConnectionsFeatureFlagOn() + throws IOException, JsonValidationException, ConfigNotFoundException, InterruptedException, ApiException { final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); final UUID connectionId = UUID.randomUUID(); final UUID connectionId2 = UUID.randomUUID(); @@ -903,7 +928,7 @@ void testDiscoverSchemaForSourceMultipleConnectionsFeatureFlagOn() throws IOExce final UUID discoveredCatalogId = UUID.randomUUID(); final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; final SourceDiscoverSchemaRequestBody request = - new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true).notifySchemaChange(true); // 3 connections use the same source. 2 will generate catalog diffs that are non-breaking, 1 will // generate a breaking catalog diff @@ -930,6 +955,9 @@ void testDiscoverSchemaForSourceMultipleConnectionsFeatureFlagOn() throws IOExce when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION), false)) .thenReturn(discoverResponse); + when(webUrlHelper.getConnectionUrl(source.getWorkspaceId(), connectionId)).thenReturn(CONNECTION_URL); + when(webUrlHelper.getConnectionUrl(source.getWorkspaceId(), connectionId2)).thenReturn(CONNECTION_URL); + when(webUrlHelper.getConnectionUrl(source.getWorkspaceId(), connectionId3)).thenReturn(CONNECTION_URL); when(discoverResponse.isSuccess()).thenReturn(true); when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); @@ -940,15 +968,15 @@ void testDiscoverSchemaForSourceMultipleConnectionsFeatureFlagOn() throws IOExce final ConnectionRead connectionRead = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).nonBreakingChangesPreference( - NonBreakingChangesPreference.IGNORE).status(ConnectionStatus.ACTIVE).connectionId(connectionId); + NonBreakingChangesPreference.IGNORE).status(ConnectionStatus.ACTIVE).connectionId(connectionId).notifySchemaChanges(true); final ConnectionRead connectionRead2 = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).nonBreakingChangesPreference( - NonBreakingChangesPreference.IGNORE).status(ConnectionStatus.ACTIVE).connectionId(connectionId2); + NonBreakingChangesPreference.IGNORE).status(ConnectionStatus.ACTIVE).connectionId(connectionId2).notifySchemaChanges(true); final ConnectionRead connectionRead3 = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent, sourceDef)).nonBreakingChangesPreference( - NonBreakingChangesPreference.DISABLE).status(ConnectionStatus.ACTIVE).connectionId(connectionId3); + NonBreakingChangesPreference.DISABLE).status(ConnectionStatus.ACTIVE).connectionId(connectionId3).notifySchemaChanges(false); when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead, connectionRead2, connectionRead3); when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff1, catalogDiff2, catalogDiff3); @@ -976,13 +1004,16 @@ void testDiscoverSchemaForSourceMultipleConnectionsFeatureFlagOn() throws IOExce assertEquals(ConnectionStatus.ACTIVE, connectionUpdateValues.get(0).getStatus()); assertEquals(ConnectionStatus.ACTIVE, connectionUpdateValues.get(1).getStatus()); assertEquals(ConnectionStatus.INACTIVE, connectionUpdateValues.get(2).getStatus()); + verify(eventRunner).sendSchemaChangeNotification(connectionId, CONNECTION_URL); + verify(eventRunner).sendSchemaChangeNotification(connectionId2, CONNECTION_URL); + verify(eventRunner, times(0)).sendSchemaChangeNotification(connectionId3, CONNECTION_URL); } @Test void testDiscoverSchemaFromSourceIdWithConnectionUpdateNonSuccessResponse() throws IOException, JsonValidationException, ConfigNotFoundException { final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); final SourceDiscoverSchemaRequestBody request = new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()) - .connectionId(UUID.randomUUID()); + .connectionId(UUID.randomUUID()).notifySchemaChange(true); // Mock the source definition. when(configRepository.getStandardSourceDefinition(source.getSourceDefinitionId())) diff --git a/airbyte-commons-temporal/build.gradle b/airbyte-commons-temporal/build.gradle index 7e553f1c7e94..2ec6401db3fd 100644 --- a/airbyte-commons-temporal/build.gradle +++ b/airbyte-commons-temporal/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':airbyte-config:config-persistence') implementation project(':airbyte-metrics:metrics-lib') implementation project(':airbyte-persistence:job-persistence') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-worker-models') implementation project(':airbyte-api') implementation project(':airbyte-json-validation') diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/NotificationUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/NotificationUtils.java new file mode 100644 index 000000000000..3af59a0214a4 --- /dev/null +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/NotificationUtils.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import io.airbyte.api.client.invoker.generated.ApiException; +import io.airbyte.commons.temporal.scheduling.ConnectionNotificationWorkflow; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.validation.json.JsonValidationException; +import io.temporal.client.WorkflowClient; +import jakarta.inject.Singleton; +import java.io.IOException; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; + +@Singleton +@Slf4j +public class NotificationUtils { + + public NotificationUtils() {} + + public void sendSchemaChangeNotification(final WorkflowClient client, final UUID connectionId, final String url) { + final ConnectionNotificationWorkflow notificationWorkflow = + client.newWorkflowStub(ConnectionNotificationWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.NOTIFY)); + try { + notificationWorkflow.sendSchemaChangeNotification(connectionId, url); + } catch (IOException | RuntimeException | InterruptedException | ApiException | ConfigNotFoundException | JsonValidationException e) { + log.error("There was an error while sending a Schema Change Notification", e); + } + } + +} diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index 4334ef422801..18409f12dd6d 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -75,6 +75,7 @@ public class TemporalClient { private final WorkflowServiceStubs service; private final StreamResetPersistence streamResetPersistence; private final ConnectionManagerUtils connectionManagerUtils; + private final NotificationUtils notificationUtils; private final StreamResetRecordsHelper streamResetRecordsHelper; public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, @@ -82,12 +83,14 @@ public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, final WorkflowServiceStubs service, final StreamResetPersistence streamResetPersistence, final ConnectionManagerUtils connectionManagerUtils, + final NotificationUtils notificationUtils, final StreamResetRecordsHelper streamResetRecordsHelper) { this.workspaceRoot = workspaceRoot; this.client = client; this.service = service; this.streamResetPersistence = streamResetPersistence; this.connectionManagerUtils = connectionManagerUtils; + this.notificationUtils = notificationUtils; this.streamResetRecordsHelper = streamResetRecordsHelper; } @@ -508,6 +511,10 @@ public void forceDeleteWorkflow(final UUID connectionId) { connectionManagerUtils.deleteWorkflowIfItExist(client, connectionId); } + public void sendSchemaChangeNotification(final UUID connectionId, final String url) { + notificationUtils.sendSchemaChangeNotification(client, connectionId, url); + } + public void update(final UUID connectionId) { final ConnectionManagerWorkflow connectionManagerWorkflow; try { diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionNotificationWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionNotificationWorkflow.java index 6005696fa962..f9035ee86d85 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionNotificationWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionNotificationWorkflow.java @@ -16,7 +16,7 @@ public interface ConnectionNotificationWorkflow { @WorkflowMethod - boolean sendSchemaChangeNotification(UUID connectionId) + boolean sendSchemaChangeNotification(UUID connectionId, String url) throws IOException, InterruptedException, ApiException, ConfigNotFoundException, JsonValidationException; } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 3b7aa1642500..f1401b79bf41 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -107,6 +107,7 @@ public class TemporalClientTest { private WorkflowServiceBlockingStub workflowServiceBlockingStub; private StreamResetPersistence streamResetPersistence; private ConnectionManagerUtils connectionManagerUtils; + private NotificationUtils notificationUtils; private StreamResetRecordsHelper streamResetRecordsHelper; private Path workspaceRoot; @@ -123,9 +124,10 @@ void setup() throws IOException { streamResetPersistence = mock(StreamResetPersistence.class); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); connectionManagerUtils = spy(new ConnectionManagerUtils()); + notificationUtils = spy(new NotificationUtils()); streamResetRecordsHelper = mock(StreamResetRecordsHelper.class); temporalClient = - spy(new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, connectionManagerUtils, + spy(new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, connectionManagerUtils, notificationUtils, streamResetRecordsHelper)); } @@ -133,13 +135,15 @@ void setup() throws IOException { class RestartPerStatus { private ConnectionManagerUtils mConnectionManagerUtils; + private NotificationUtils mNotificationUtils; @BeforeEach void init() { mConnectionManagerUtils = mock(ConnectionManagerUtils.class); + mNotificationUtils = mock(NotificationUtils.class); temporalClient = spy( - new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils, + new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils, mNotificationUtils, streamResetRecordsHelper)); } diff --git a/airbyte-commons-worker/build.gradle b/airbyte-commons-worker/build.gradle index 7b64271baac2..8b46f830a30d 100644 --- a/airbyte-commons-worker/build.gradle +++ b/airbyte-commons-worker/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation project(':airbyte-json-validation') implementation project(':airbyte-metrics:metrics-lib') implementation project(':airbyte-persistence:job-persistence') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-worker-models') testAnnotationProcessor platform(libs.micronaut.bom) diff --git a/airbyte-config/config-models/build.gradle b/airbyte-config/config-models/build.gradle index c6e73c63d565..174775124c02 100644 --- a/airbyte-config/config-models/build.gradle +++ b/airbyte-config/config-models/build.gradle @@ -10,7 +10,7 @@ dependencies { api libs.bundles.micronaut.annotation implementation project(':airbyte-json-validation') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-commons') } diff --git a/airbyte-config/config-persistence/build.gradle b/airbyte-config/config-persistence/build.gradle index 48c2772aa34e..d43e903ab3c2 100644 --- a/airbyte-config/config-persistence/build.gradle +++ b/airbyte-config/config-persistence/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-db:jooq') implementation project(':airbyte-json-validation') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-metrics:metrics-lib') implementation 'commons-io:commons-io:2.7' diff --git a/airbyte-config/init/build.gradle b/airbyte-config/init/build.gradle index 5de81b22861d..887ef2adc0e0 100644 --- a/airbyte-config/init/build.gradle +++ b/airbyte-config/init/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') implementation project(':airbyte-persistence:job-persistence') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-json-validation') implementation libs.lombok implementation libs.micronaut.cache.caffeine diff --git a/airbyte-config/specs/build.gradle b/airbyte-config/specs/build.gradle index 123fee561a37..a7800954fc80 100644 --- a/airbyte-config/specs/build.gradle +++ b/airbyte-config/specs/build.gradle @@ -8,7 +8,7 @@ dependencies { implementation project(':airbyte-commons') implementation project(':airbyte-commons-cli') implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-json-validation') } diff --git a/airbyte-connector-builder-server/run_tests.sh b/airbyte-connector-builder-server/run_tests.sh index a82c60d452bc..d84818013941 100755 --- a/airbyte-connector-builder-server/run_tests.sh +++ b/airbyte-connector-builder-server/run_tests.sh @@ -6,10 +6,9 @@ cd $1 set -e # Install dependencies -pip install -e . -pip install -e '.[main]' -pip install -e '.[tests]' +pip install -q -e . +pip install -q -e '.[tests]' # Run the tests -python -m coverage run -m pytest unit_tests -c pytest.ini -python -m coverage run -m pytest integration_tests -c pytest.ini +python -m coverage run -m pytest -p no:logging --disable-warnings unit_tests -c pytest.ini +python -m coverage run -m pytest -p no:logging --disable-warnings integration_tests -c pytest.ini diff --git a/airbyte-container-orchestrator/build.gradle b/airbyte-container-orchestrator/build.gradle index 473aae7e5eed..835e34290650 100644 --- a/airbyte-container-orchestrator/build.gradle +++ b/airbyte-container-orchestrator/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation project(':airbyte-config:init') implementation project(':airbyte-featureflag') implementation project(':airbyte-json-validation') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-metrics:metrics-lib') implementation project(':airbyte-worker-models') diff --git a/airbyte-db/db-lib/build.gradle b/airbyte-db/db-lib/build.gradle index 33ec3409432a..1386fe905006 100644 --- a/airbyte-db/db-lib/build.gradle +++ b/airbyte-db/db-lib/build.gradle @@ -17,7 +17,7 @@ dependencies { api libs.jooq api libs.postgresql - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-json-validation') implementation project(':airbyte-config:config-models') implementation libs.flyway.core diff --git a/airbyte-integrations/bases/base-java-s3/build.gradle b/airbyte-integrations/bases/base-java-s3/build.gradle index f3700344a639..73780c0dd6f6 100644 --- a/airbyte-integrations/bases/base-java-s3/build.gradle +++ b/airbyte-integrations/bases/base-java-s3/build.gradle @@ -4,7 +4,7 @@ plugins { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index f3a90f553338..45d66f5e2adb 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -4,7 +4,7 @@ plugins { } dependencies { - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-config:config-models') implementation project(':airbyte-commons-cli') implementation project(':airbyte-json-validation') diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index 1bb67ccc67f2..da91d1a3e142 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -30,5 +30,5 @@ ENTRYPOINT ["/airbyte/entrypoint.sh"] # 0.3.x is tombstoned. # The next minor bump should go directly to 0.4.0 -LABEL io.airbyte.version=0.2.25 +LABEL io.airbyte.version=0.2.26 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py index 158d4307a689..5c55b776c67b 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, Set import yaml -from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode # type: ignore from normalization.destination_type import DestinationType from normalization.transform_catalog import dbt_macro from normalization.transform_catalog.destination_name_transformer import DestinationNameTransformer diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 986beb92b2f0..8ca26c72b5b1 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -8,7 +8,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Tuple, Union -from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode # type: ignore from jinja2 import Template from normalization.destination_type import DestinationType from normalization.transform_catalog import dbt_macro diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index b27caa87182d..7c14e02f6490 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -325,10 +325,14 @@ def transform_clickhouse(config: Dict[str, Any]): "port": config["port"], "schema": config["database"], "user": config["username"], - "secure": config["ssl"], } if "password" in config: dbt_config["password"] = config["password"] + + # ssl is an optional configuration and is not present in strict-encrypt config + # if ssl option is not present in the config - default to True + dbt_config["secure"] = config.get("ssl", True) + return dbt_config @staticmethod diff --git a/airbyte-integrations/bases/base-standard-source-test-file/build.gradle b/airbyte-integrations/bases/base-standard-source-test-file/build.gradle index 4559d3b7580d..f58675f3de08 100644 --- a/airbyte-integrations/bases/base-standard-source-test-file/build.gradle +++ b/airbyte-integrations/bases/base-standard-source-test-file/build.gradle @@ -4,7 +4,7 @@ plugins { } dependencies { - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:standard-source-test') implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' diff --git a/airbyte-integrations/bases/debezium-v1-9-6/build.gradle b/airbyte-integrations/bases/debezium-v1-9-6/build.gradle index 633cb787a57d..e33738b17f40 100644 --- a/airbyte-integrations/bases/debezium-v1-9-6/build.gradle +++ b/airbyte-integrations/bases/debezium-v1-9-6/build.gradle @@ -6,7 +6,7 @@ project.configurations { testFixturesImplementation.extendsFrom implementation } dependencies { - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-db:db-lib') implementation 'io.debezium:debezium-api:1.9.6.Final' diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle b/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle index 3a0c06738db8..779f9a199e6b 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle +++ b/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle @@ -4,7 +4,7 @@ plugins { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') diff --git a/airbyte-integrations/bases/standard-destination-test/build.gradle b/airbyte-integrations/bases/standard-destination-test/build.gradle index fc0f369fdbbf..a6ef45eb9abc 100644 --- a/airbyte-integrations/bases/standard-destination-test/build.gradle +++ b/airbyte-integrations/bases/standard-destination-test/build.gradle @@ -8,7 +8,7 @@ dependencies { implementation project(':airbyte-config:init') implementation project(':airbyte-json-validation') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation(enforcedPlatform('org.junit:junit-bom:5.8.2')) implementation 'org.junit.jupiter:junit-jupiter-api' diff --git a/airbyte-integrations/bases/standard-source-test/build.gradle b/airbyte-integrations/bases/standard-source-test/build.gradle index 2ea4836076fc..9d64af20a834 100644 --- a/airbyte-integrations/bases/standard-source-test/build.gradle +++ b/airbyte-integrations/bases/standard-source-test/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':airbyte-commons-worker') implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation 'org.mockito:mockito-core:4.6.1' implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' diff --git a/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs b/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs index 3c6c4d037882..7ae9054c8e0e 100644 --- a/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs +++ b/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs @@ -10,7 +10,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle b/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle index cffa4c5b0bbd..cd68e6d2ad45 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle +++ b/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -27,4 +27,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle b/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle index 1e1882ebb22e..07e93c50bbf0 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle index 8f7e0786c3ee..1f837dc6eaf0 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':airbyte-config:config-models') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:destination-bigquery') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java-s3') implementation project(':airbyte-integrations:connectors:destination-gcs') implementation group: 'org.apache.parquet', name: 'parquet-avro', version: '1.12.0' diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 42f989556d60..107c606c5482 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation project(':airbyte-config:config-models') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java-s3') implementation project(':airbyte-integrations:connectors:destination-gcs') implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} diff --git a/airbyte-integrations/connectors/destination-cassandra/build.gradle b/airbyte-integrations/connectors/destination-cassandra/build.gradle index b5cca5c52e3f..f8cb0ca16b89 100644 --- a/airbyte-integrations/connectors/destination-cassandra/build.gradle +++ b/airbyte-integrations/connectors/destination-cassandra/build.gradle @@ -14,7 +14,7 @@ def assertVersion = '3.21.0' dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index 5e0280c4d223..2535c3b4a5fe 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-clickhouse') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index e1e41edb6496..40b2ece6b7d0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-csv/build.gradle b/airbyte-integrations/connectors/destination-csv/build.gradle index 56219604025a..5d02a01f74ee 100644 --- a/airbyte-integrations/connectors/destination-csv/build.gradle +++ b/airbyte-integrations/connectors/destination-csv/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation 'org.apache.commons:commons-csv:1.4' implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-databricks/build.gradle b/airbyte-integrations/connectors/destination-databricks/build.gradle index 4ca5d1e85fc8..4829bc8d287d 100644 --- a/airbyte-integrations/connectors/destination-databricks/build.gradle +++ b/airbyte-integrations/connectors/destination-databricks/build.gradle @@ -27,7 +27,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-dev-null/build.gradle b/airbyte-integrations/connectors/destination-dev-null/build.gradle index a07b667da17a..5882aa1f045e 100644 --- a/airbyte-integrations/connectors/destination-dev-null/build.gradle +++ b/airbyte-integrations/connectors/destination-dev-null/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:destination-e2e-test') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-doris/build.gradle b/airbyte-integrations/connectors/destination-doris/build.gradle index 82e4c702251f..3639f240c70b 100644 --- a/airbyte-integrations/connectors/destination-doris/build.gradle +++ b/airbyte-integrations/connectors/destination-doris/build.gradle @@ -12,7 +12,7 @@ dependencies { implementation 'org.apache.commons:commons-csv:1.4' implementation group: 'mysql', name: 'mysql-connector-java', version: '8.0.16' implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-dynamodb/build.gradle b/airbyte-integrations/connectors/destination-dynamodb/build.gradle index ecfbb13e5710..db63c3c00960 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/destination-dynamodb/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-e2e-test/build.gradle b/airbyte-integrations/connectors/destination-e2e-test/build.gradle index 0c8ce15202de..8d6111cf0512 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/destination-e2e-test/build.gradle @@ -10,7 +10,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle index 77da0535c523..07ad7bf8cc36 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle index 469ad4991a16..37eb3cbf3882 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index 9d37cc4cc2d6..a90eceeb8294 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') implementation project(':airbyte-integrations:connectors:destination-jdbc') diff --git a/airbyte-integrations/connectors/destination-iceberg/build.gradle b/airbyte-integrations/connectors/destination-iceberg/build.gradle index 13fe5ff1d923..f08f5561e0f5 100644 --- a/airbyte-integrations/connectors/destination-iceberg/build.gradle +++ b/airbyte-integrations/connectors/destination-iceberg/build.gradle @@ -10,7 +10,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) @@ -51,4 +51,4 @@ test { task prepareKotlinBuildScriptModel { -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-jdbc/build.gradle b/airbyte-integrations/connectors/destination-jdbc/build.gradle index a20bef21b611..57c98181a3e9 100644 --- a/airbyte-integrations/connectors/destination-jdbc/build.gradle +++ b/airbyte-integrations/connectors/destination-jdbc/build.gradle @@ -11,7 +11,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation 'org.apache.commons:commons-lang3:3.11' implementation 'org.apache.commons:commons-csv:1.4' diff --git a/airbyte-integrations/connectors/destination-kafka/build.gradle b/airbyte-integrations/connectors/destination-kafka/build.gradle index 001dcfc35c76..60a0c8b0f23a 100644 --- a/airbyte-integrations/connectors/destination-kafka/build.gradle +++ b/airbyte-integrations/connectors/destination-kafka/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation 'org.apache.kafka:kafka-clients:2.8.0' diff --git a/airbyte-integrations/connectors/destination-keen/build.gradle b/airbyte-integrations/connectors/destination-keen/build.gradle index 4c35e9daea8a..6e61e5b4738c 100644 --- a/airbyte-integrations/connectors/destination-keen/build.gradle +++ b/airbyte-integrations/connectors/destination-keen/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation 'org.apache.kafka:kafka-clients:2.8.0' diff --git a/airbyte-integrations/connectors/destination-kinesis/build.gradle b/airbyte-integrations/connectors/destination-kinesis/build.gradle index b92d73d3f8d4..676ce90ab007 100644 --- a/airbyte-integrations/connectors/destination-kinesis/build.gradle +++ b/airbyte-integrations/connectors/destination-kinesis/build.gradle @@ -15,7 +15,7 @@ def assertVersion = '3.21.0' dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-local-json/build.gradle b/airbyte-integrations/connectors/destination-local-json/build.gradle index f2c4761f1af0..5645c9171058 100644 --- a/airbyte-integrations/connectors/destination-local-json/build.gradle +++ b/airbyte-integrations/connectors/destination-local-json/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle b/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle index 2b0fbea1d9bb..139b685bc62d 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle index 287ac8afe5af..0a02550bb278 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-config:config-models') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-mongodb') implementation 'org.mongodb:mongodb-driver-sync:4.3.0' diff --git a/airbyte-integrations/connectors/destination-mongodb/build.gradle b/airbyte-integrations/connectors/destination-mongodb/build.gradle index d84458837649..9361454f02ad 100644 --- a/airbyte-integrations/connectors/destination-mongodb/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-config:config-models') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation 'org.mongodb:mongodb-driver-sync:4.3.0' diff --git a/airbyte-integrations/connectors/destination-mqtt/build.gradle b/airbyte-integrations/connectors/destination-mqtt/build.gradle index 0fbdbadf1fac..a2cc814a7594 100644 --- a/airbyte-integrations/connectors/destination-mqtt/build.gradle +++ b/airbyte-integrations/connectors/destination-mqtt/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle index 3f12dcac7cc2..c5273bda2103 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation project(':airbyte-config:config-models') implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-mssql') implementation project(':airbyte-test-utils') diff --git a/airbyte-integrations/connectors/destination-mssql/build.gradle b/airbyte-integrations/connectors/destination-mssql/build.gradle index ff95ba711b84..b496d19b93c2 100644 --- a/airbyte-integrations/connectors/destination-mssql/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-test-utils') diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle index 2b28507b7a42..dbdb228986b9 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-mysql') @@ -25,4 +25,3 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } - diff --git a/airbyte-integrations/connectors/destination-mysql/build.gradle b/airbyte-integrations/connectors/destination-mysql/build.gradle index 9434a5c06812..5c27cc30ff09 100644 --- a/airbyte-integrations/connectors/destination-mysql/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation 'mysql:mysql-connector-java:8.0.22' @@ -24,4 +24,3 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } - diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle index 58f5ff7cf493..5e92820dc4c5 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-oracle') diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index df29788e3aa8..40fb19846581 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle index 5e7fa9bf897e..1e4704b3da54 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-postgres') diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index 06a8e9f2a69a..cd2b73246fe5 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') testImplementation project(':airbyte-test-utils') diff --git a/airbyte-integrations/connectors/destination-pubsub/build.gradle b/airbyte-integrations/connectors/destination-pubsub/build.gradle index 14dc95279e67..767a4af8d40b 100644 --- a/airbyte-integrations/connectors/destination-pubsub/build.gradle +++ b/airbyte-integrations/connectors/destination-pubsub/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '1.114.7' implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-pulsar/build.gradle b/airbyte-integrations/connectors/destination-pulsar/build.gradle index 6e12ff7c64e5..e308c25d2714 100644 --- a/airbyte-integrations/connectors/destination-pulsar/build.gradle +++ b/airbyte-integrations/connectors/destination-pulsar/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-r2/build.gradle b/airbyte-integrations/connectors/destination-r2/build.gradle index 834de43392a9..07c8db9c43f6 100644 --- a/airbyte-integrations/connectors/destination-r2/build.gradle +++ b/airbyte-integrations/connectors/destination-r2/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-redis/build.gradle b/airbyte-integrations/connectors/destination-redis/build.gradle index 4f59448f5ed4..58fca9472caa 100644 --- a/airbyte-integrations/connectors/destination-redis/build.gradle +++ b/airbyte-integrations/connectors/destination-redis/build.gradle @@ -14,7 +14,7 @@ def assertVersion = '3.21.0' dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-redpanda/build.gradle b/airbyte-integrations/connectors/destination-redpanda/build.gradle index 72006ed6beb7..9089c8e7c242 100644 --- a/airbyte-integrations/connectors/destination-redpanda/build.gradle +++ b/airbyte-integrations/connectors/destination-redpanda/build.gradle @@ -10,7 +10,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index ca61023831dc..8cc09fee330f 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -18,7 +18,7 @@ repositories { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-integrations:bases:base-java-s3') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-rockset/build.gradle b/airbyte-integrations/connectors/destination-rockset/build.gradle index 2bd1e3ecb051..80718c4c552d 100644 --- a/airbyte-integrations/connectors/destination-rockset/build.gradle +++ b/airbyte-integrations/connectors/destination-rockset/build.gradle @@ -16,7 +16,7 @@ repositories { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation project(':airbyte-integrations:connectors:destination-jdbc') diff --git a/airbyte-integrations/connectors/destination-s3-glue/build.gradle b/airbyte-integrations/connectors/destination-s3-glue/build.gradle index 9499fbc68045..a8aca0f9987a 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/build.gradle +++ b/airbyte-integrations/connectors/destination-s3-glue/build.gradle @@ -10,7 +10,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index dcdf8e668aa1..12f1c19bb084 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-scylla/build.gradle b/airbyte-integrations/connectors/destination-scylla/build.gradle index 21d45bb766fb..236e005f6c45 100644 --- a/airbyte-integrations/connectors/destination-scylla/build.gradle +++ b/airbyte-integrations/connectors/destination-scylla/build.gradle @@ -14,7 +14,7 @@ def assertVersion = '3.21.0' dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index 98abaec31d1e..c9287b5f2d89 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -42,7 +42,7 @@ dependencies { implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-gcs') implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol // this is a configuration to make mockito work with final classes testImplementation 'org.mockito:mockito-inline:2.13.0' diff --git a/airbyte-integrations/connectors/destination-teradata/build.gradle b/airbyte-integrations/connectors/destination-teradata/build.gradle index c755247f5bc8..bfab73ae6fd8 100644 --- a/airbyte-integrations/connectors/destination-teradata/build.gradle +++ b/airbyte-integrations/connectors/destination-teradata/build.gradle @@ -11,16 +11,15 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation 'com.teradata.jdbc:terajdbc4:17.20.00.12' integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-teradata') } - diff --git a/airbyte-integrations/connectors/destination-tidb/build.gradle b/airbyte-integrations/connectors/destination-tidb/build.gradle index 994c991abb28..bfe578b3dfc2 100644 --- a/airbyte-integrations/connectors/destination-tidb/build.gradle +++ b/airbyte-integrations/connectors/destination-tidb/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-jdbc') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-yugabytedb/build.gradle b/airbyte-integrations/connectors/destination-yugabytedb/build.gradle index 51ae7a928363..2d0d18ad7253 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/build.gradle +++ b/airbyte-integrations/connectors/destination-yugabytedb/build.gradle @@ -10,7 +10,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation project(':airbyte-integrations:connectors:destination-jdbc') diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/build.gradle index 4fd2aeb559ac..5cab3833daab 100644 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-postgres') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -28,4 +28,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-alloydb/build.gradle b/airbyte-integrations/connectors/source-alloydb/build.gradle index 4e4bd028d8c0..a92d92a5e8b9 100644 --- a/airbyte-integrations/connectors/source-alloydb/build.gradle +++ b/airbyte-integrations/connectors/source-alloydb/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-postgres') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -30,4 +30,4 @@ dependencies { integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-bigquery/build.gradle b/airbyte-integrations/connectors/source-bigquery/build.gradle index 5f32df98c2e9..4aed77faa3ee 100644 --- a/airbyte-integrations/connectors/source-bigquery/build.gradle +++ b/airbyte-integrations/connectors/source-bigquery/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation 'org.apache.commons:commons-lang3:3.11' implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -28,4 +28,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle index 913c114460af..f1d00512e126 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') implementation project(':airbyte-integrations:connectors:source-clickhouse') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation group: 'com.clickhouse', name: 'clickhouse-jdbc', version: '0.3.2-patch9' diff --git a/airbyte-integrations/connectors/source-clickhouse/build.gradle b/airbyte-integrations/connectors/source-clickhouse/build.gradle index dca21deacd5d..5e9bb7f0f79c 100644 --- a/airbyte-integrations/connectors/source-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/build.gradle index ce46931f0410..f651236698cc 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') implementation project(':airbyte-integrations:connectors:source-cockroachdb') @@ -28,4 +28,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-cockroachdb/build.gradle b/airbyte-integrations/connectors/source-cockroachdb/build.gradle index 2e4e4c7d280d..172d69b97a31 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/build.gradle +++ b/airbyte-integrations/connectors/source-cockroachdb/build.gradle @@ -13,7 +13,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -34,4 +34,4 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation project(":airbyte-json-validation") testImplementation project(':airbyte-test-utils') -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle index 2c16590255c4..62cedc738ebe 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation group: 'com.ibm.db2', name: 'jcc', version: '11.5.5.0' diff --git a/airbyte-integrations/connectors/source-db2/build.gradle b/airbyte-integrations/connectors/source-db2/build.gradle index 9b8277db39b0..5a7edbffc579 100644 --- a/airbyte-integrations/connectors/source-db2/build.gradle +++ b/airbyte-integrations/connectors/source-db2/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation group: 'com.ibm.db2', name: 'jcc', version: '11.5.5.0' diff --git a/airbyte-integrations/connectors/source-dynamodb/build.gradle b/airbyte-integrations/connectors/source-dynamodb/build.gradle index c6b41ba2e3bf..4689dbb3da97 100644 --- a/airbyte-integrations/connectors/source-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/source-dynamodb/build.gradle @@ -14,7 +14,7 @@ def assertVersion = '3.23.1' dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-relational-db') implementation project(':airbyte-config:config-models') @@ -40,4 +40,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle b/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle index 5fd0c784d514..f05eab45d34c 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle @@ -10,7 +10,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-e2e-test') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/source-e2e-test/build.gradle b/airbyte-integrations/connectors/source-e2e-test/build.gradle index b532f8db18da..ba548f2a568d 100644 --- a/airbyte-integrations/connectors/source-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/source-e2e-test/build.gradle @@ -10,7 +10,7 @@ application { dependencies { implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-json-validation') implementation 'org.apache.commons:commons-lang3:3.11' implementation 'com.networknt:json-schema-validator:1.0.72' diff --git a/airbyte-integrations/connectors/source-elasticsearch/build.gradle b/airbyte-integrations/connectors/source-elasticsearch/build.gradle index b1c29c72294d..0997d855f17b 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/source-elasticsearch/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/source-jdbc/build.gradle b/airbyte-integrations/connectors/source-jdbc/build.gradle index 777620be980a..cb142aeab711 100644 --- a/airbyte-integrations/connectors/source-jdbc/build.gradle +++ b/airbyte-integrations/connectors/source-jdbc/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation project(':airbyte-commons') implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-relational-db') implementation 'org.apache.commons:commons-lang3:3.11' @@ -37,7 +37,7 @@ dependencies { integrationTestJavaImplementation libs.connectors.testcontainers.postgresql testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" - testFixturesImplementation project(':airbyte-protocol:protocol-models') + testFixturesImplementation libs.airbyte.protocol testFixturesImplementation project(':airbyte-db:db-lib') testFixturesImplementation project(':airbyte-integrations:bases:base-java') diff --git a/airbyte-integrations/connectors/source-kafka/build.gradle b/airbyte-integrations/connectors/source-kafka/build.gradle index 202bf2abfea8..24c49ba0ad1c 100644 --- a/airbyte-integrations/connectors/source-kafka/build.gradle +++ b/airbyte-integrations/connectors/source-kafka/build.gradle @@ -21,7 +21,7 @@ repositories { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation libs.connectors.testcontainers.kafka diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index d0adc8d7719f..c63a37914524 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -332,7 +332,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> List[str]: class Leads(MarketoExportBase): """ Return list of all leeds. - API Docs: http://developers.marketo.com/rest-api/bulk-extract/bulk-lead-extract/ + API Docs: https://developers.marketo.com/rest-api/bulk-extract/bulk-lead-extract/ """ cursor_field = "updatedAt" @@ -349,7 +349,7 @@ class Activities(MarketoExportBase): """ Base class for all the activities streams, provides functionality for dynamically created classes as streams of data. - API Docs: http://developers.marketo.com/rest-api/bulk-extract/bulk-activity-extract/ + API Docs: https://developers.marketo.com/rest-api/bulk-extract/bulk-activity-extract/ """ primary_key = "marketoGUID" @@ -411,7 +411,7 @@ def get_json_schema(self) -> Mapping[str, Any]: class ActivityTypes(MarketoStream): """ Return list of all activity types. - API Docs: http://developers.marketo.com/rest-api/lead-database/activities/#describe + API Docs: https://developers.marketo.com/rest-api/lead-database/activities/#describe """ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -421,7 +421,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Programs(IncrementalMarketoStream): """ Return list of all programs. - API Docs: http://developers.marketo.com/rest-api/assets/programs/#by_date_range + API Docs: https://developers.marketo.com/rest-api/assets/programs/#by_date_range """ cursor_field = "updatedAt" diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle index a346b3880bb4..6adacf948ad8 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation project(':airbyte-integrations:connectors:source-relational-db') implementation project(':airbyte-integrations:connectors:source-mongodb-v2') diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index 184450a14c73..896e88e74958 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -13,7 +13,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation project(':airbyte-integrations:connectors:source-relational-db') diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle index eb2077c7d275..82d2b6f983df 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-jdbc') diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle index ffd490be4ee4..601c5306023b 100644 --- a/airbyte-integrations/connectors/source-mssql/build.gradle +++ b/airbyte-integrations/connectors/source-mssql/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:debezium-v1-9-6') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle index 99eb1b7378d4..ddb4516e6365 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-mysql') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 89f6eb87b0f7..7d6ab67ea010 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:debezium-v1-9-6') implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-relational-db') implementation 'mysql:mysql-connector-java:8.0.30' diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle index 69ae71a6f8cc..af6d4228c3d1 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-oracle') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') diff --git a/airbyte-integrations/connectors/source-oracle/build.gradle b/airbyte-integrations/connectors/source-oracle/build.gradle index e5dfe20d3995..9bf6b68b10b8 100644 --- a/airbyte-integrations/connectors/source-oracle/build.gradle +++ b/airbyte-integrations/connectors/source-oracle/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle index 639a31c370be..5358d9820cb2 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-postgres') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol // todo (cgardens): why are these needed? implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -29,4 +29,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index 71f93708d1f5..34fb3785c894 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation 'io.debezium:debezium-embedded:1.9.6.Final' implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:debezium-v1-9-6') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') diff --git a/airbyte-integrations/connectors/source-redshift/build.gradle b/airbyte-integrations/connectors/source-redshift/build.gradle index 68dc0b6c8640..baa9c4244af6 100644 --- a/airbyte-integrations/connectors/source-redshift/build.gradle +++ b/airbyte-integrations/connectors/source-redshift/build.gradle @@ -17,7 +17,7 @@ repositories { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') diff --git a/airbyte-integrations/connectors/source-relational-db/build.gradle b/airbyte-integrations/connectors/source-relational-db/build.gradle index 8d9b66c70cad..a58ae385673e 100644 --- a/airbyte-integrations/connectors/source-relational-db/build.gradle +++ b/airbyte-integrations/connectors/source-relational-db/build.gradle @@ -9,7 +9,7 @@ dependencies { implementation project(':airbyte-commons') implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-json-validation') implementation project(':airbyte-config:config-models') diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle b/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle index 06fee4a3b8f4..dbbe4f113e84 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle @@ -12,7 +12,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -27,4 +27,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-sftp/build.gradle b/airbyte-integrations/connectors/source-sftp/build.gradle index d416e7cfca52..7dfbd6003034 100644 --- a/airbyte-integrations/connectors/source-sftp/build.gradle +++ b/airbyte-integrations/connectors/source-sftp/build.gradle @@ -11,7 +11,7 @@ application { dependencies { implementation project(':airbyte-config:config-models') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.13.2' diff --git a/airbyte-integrations/connectors/source-snowflake/build.gradle b/airbyte-integrations/connectors/source-snowflake/build.gradle index cff90793c768..1a75eced1fd5 100644 --- a/airbyte-integrations/connectors/source-snowflake/build.gradle +++ b/airbyte-integrations/connectors/source-snowflake/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation group: 'net.snowflake', name: 'snowflake-jdbc', version: '3.13.22' implementation 'com.zaxxer:HikariCP:5.0.1' diff --git a/airbyte-integrations/connectors/source-tidb/build.gradle b/airbyte-integrations/connectors/source-tidb/build.gradle index 5b7b1ca5f128..b4f9f28d3ee5 100644 --- a/airbyte-integrations/connectors/source-tidb/build.gradle +++ b/airbyte-integrations/connectors/source-tidb/build.gradle @@ -13,7 +13,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java index ae284e80f86b..279625c280ae 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java @@ -7,6 +7,7 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.resources.MoreResources; import io.airbyte.config.Notification; +import io.airbyte.config.SlackNotificationConfiguration; import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -122,7 +123,10 @@ public boolean notifyFailure(final String message) throws IOException, Interrupt } @Override - public boolean notifySchemaChange(final UUID connectionId, final boolean isBreaking) { + public boolean notifySchemaChange(final UUID connectionId, + final boolean isBreaking, + final SlackNotificationConfiguration config, + final String url) { throw new NotImplementedException(); } diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java index 6d6850feb459..b723efe45fc8 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java @@ -6,6 +6,7 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.config.Notification; +import io.airbyte.config.SlackNotificationConfiguration; import java.io.IOException; import java.util.UUID; @@ -55,7 +56,11 @@ public abstract boolean notifyConnectionDisableWarning(String receiverEmail, public abstract boolean notifyFailure(String message) throws IOException, InterruptedException; - public abstract boolean notifySchemaChange(UUID connectionId, boolean isBreaking) throws IOException, InterruptedException; + public abstract boolean notifySchemaChange(final UUID connectionId, + final boolean isBreaking, + final SlackNotificationConfiguration config, + final String url) + throws IOException, InterruptedException; public static NotificationClient createNotificationClient(final Notification notification) { return switch (notification.getNotificationType()) { diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java index 98616fd0613c..a93902713592 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java @@ -121,10 +121,12 @@ public boolean notifyConnectionDisableWarning(final String receiverEmail, } @Override - public boolean notifySchemaChange(UUID connectionId, boolean isBreaking) throws IOException, InterruptedException { + public boolean notifySchemaChange(UUID connectionId, boolean isBreaking, SlackNotificationConfiguration config, String url) + throws IOException, InterruptedException { final String message = renderTemplate( - isBreaking ? "slack/breaking_schema_change_notification_template.txt" : "slack/non_breaking_schema_change_notification_template.txt", - connectionId.toString()); + isBreaking ? "slack/breaking_schema_change_slack_notification_template.txt" + : "slack/non_breaking_schema_change_slack_notification_template.txt", + connectionId.toString(), url); final String webhookUrl = config.getWebhook(); if (!Strings.isEmpty(webhookUrl)) { return notify(message); diff --git a/airbyte-notification/src/main/resources/slack/breaking_schema_change_notification_template.txt b/airbyte-notification/src/main/resources/slack/breaking_schema_change_notification_template.txt deleted file mode 100644 index 925089685ffc..000000000000 --- a/airbyte-notification/src/main/resources/slack/breaking_schema_change_notification_template.txt +++ /dev/null @@ -1,3 +0,0 @@ -Your source schema has changed for connection ID: %s - -Airbyte has disabled this connection because this source schema change will cause broken syncs. Visit your connection page, refresh your source schema, and reset your data in order to fix this connection. diff --git a/airbyte-notification/src/main/resources/slack/breaking_schema_change_slack_notification_template.txt b/airbyte-notification/src/main/resources/slack/breaking_schema_change_slack_notification_template.txt new file mode 100644 index 000000000000..a20974f3fa55 --- /dev/null +++ b/airbyte-notification/src/main/resources/slack/breaking_schema_change_slack_notification_template.txt @@ -0,0 +1,7 @@ +Your source schema has changed for connection ID: %s + +Airbyte has disabled this connection because this source schema change will cause broken syncs. + +Visit your connection page here: %s + +Refresh your source schema and reset your data in order to fix this connection. diff --git a/airbyte-notification/src/main/resources/slack/non_breaking_schema_change_notification_template.txt b/airbyte-notification/src/main/resources/slack/non_breaking_schema_change_notification_template.txt deleted file mode 100644 index 79c8447abbd3..000000000000 --- a/airbyte-notification/src/main/resources/slack/non_breaking_schema_change_notification_template.txt +++ /dev/null @@ -1,3 +0,0 @@ -Your source schema has changed for connection ID: %s - -Visit your connection page, refresh your source schema, and reset your data in order to update this connection. diff --git a/airbyte-notification/src/main/resources/slack/non_breaking_schema_change_slack_notification_template.txt b/airbyte-notification/src/main/resources/slack/non_breaking_schema_change_slack_notification_template.txt new file mode 100644 index 000000000000..4f32a9a30a5d --- /dev/null +++ b/airbyte-notification/src/main/resources/slack/non_breaking_schema_change_slack_notification_template.txt @@ -0,0 +1,5 @@ +Your source schema has changed for connection ID: %s + +Visit your connection page here: %s + +Refresh your source schema in order to update this connection. diff --git a/airbyte-oauth/build.gradle b/airbyte-oauth/build.gradle index 41df583abef7..e13237785b56 100644 --- a/airbyte-oauth/build.gradle +++ b/airbyte-oauth/build.gradle @@ -7,7 +7,7 @@ dependencies { implementation project(':airbyte-config:config-models') implementation project(':airbyte-config:config-persistence') implementation project(':airbyte-json-validation') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol testImplementation project(':airbyte-oauth') } diff --git a/airbyte-persistence/job-persistence/build.gradle b/airbyte-persistence/job-persistence/build.gradle index 3ae3648b5488..3bc8f7e29f90 100644 --- a/airbyte-persistence/job-persistence/build.gradle +++ b/airbyte-persistence/job-persistence/build.gradle @@ -9,7 +9,7 @@ dependencies { implementation project(':airbyte-config:config-models') implementation project(':airbyte-db:jooq') implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-config:config-persistence') implementation project(':airbyte-json-validation') implementation project(':airbyte-notification') diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 3d4ea0c87367..d88d9e30f5ff 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -45,7 +45,7 @@ dependencies { implementation project(":airbyte-json-validation") implementation project(':airbyte-notification') implementation project(':airbyte-oauth') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-persistence:job-persistence') implementation 'com.github.slugify:slugify:2.4' diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 035fa682cab0..6cb04bf40dc9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -18,6 +18,7 @@ import io.airbyte.commons.server.scheduler.TemporalEventRunner; import io.airbyte.commons.server.services.AirbyteGithubStore; import io.airbyte.commons.temporal.ConnectionManagerUtils; +import io.airbyte.commons.temporal.NotificationUtils; import io.airbyte.commons.temporal.StreamResetRecordsHelper; import io.airbyte.commons.temporal.TemporalClient; import io.airbyte.commons.temporal.TemporalUtils; @@ -49,6 +50,7 @@ import io.airbyte.persistence.job.tracker.JobTracker; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.workers.helper.ConnectionHelper; +import io.temporal.client.WorkflowClient; import io.temporal.serviceclient.WorkflowServiceStubs; import java.net.http.HttpClient; import java.util.Map; @@ -213,14 +215,17 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, final StreamResetPersistence streamResetPersistence = new StreamResetPersistence(configsDatabase); final WorkflowServiceStubs temporalService = temporalUtils.createTemporalService(); final ConnectionManagerUtils connectionManagerUtils = new ConnectionManagerUtils(); + final NotificationUtils notificationUtils = new NotificationUtils(); final StreamResetRecordsHelper streamResetRecordsHelper = new StreamResetRecordsHelper(jobPersistence, streamResetPersistence); + final WorkflowClient workflowClient = TemporalWorkflowUtils.createWorkflowClient(temporalService, temporalUtils.getNamespace()); final TemporalClient temporalClient = new TemporalClient( configs.getWorkspaceRoot(), - TemporalWorkflowUtils.createWorkflowClient(temporalService, temporalUtils.getNamespace()), + workflowClient, temporalService, streamResetPersistence, connectionManagerUtils, + notificationUtils, streamResetRecordsHelper); final OAuthConfigSupplier oAuthConfigSupplier = new OAuthConfigSupplier(configRepository, trackingClient); @@ -265,7 +270,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, configs.getLogConfigs(), eventRunner, connectionsHandler, - envVariableFeatureFlags); + envVariableFeatureFlags, + webUrlHelper); final AirbyteProtocolVersionRange airbyteProtocolVersionRange = new AirbyteProtocolVersionRange(configs.getAirbyteProtocolVersionMin(), configs.getAirbyteProtocolVersionMax()); diff --git a/airbyte-webapp-e2e-tests/cypress/commands/db/queries.ts b/airbyte-webapp-e2e-tests/cypress/commands/db/queries.ts index bf7128cf2cbf..a05c742fad39 100644 --- a/airbyte-webapp-e2e-tests/cypress/commands/db/queries.ts +++ b/airbyte-webapp-e2e-tests/cypress/commands/db/queries.ts @@ -66,3 +66,23 @@ export const createCarsTableQuery = createTable("public.cars", [ ]); export const dropCarsTableQuery = dropTable("public.cars"); + +// Dummy tables - used only for populating stream table with a lot of streams(tables) +// NOTE: Not for testing stream functionality! +export const createDummyTablesQuery = (amountOfTables: number) => + Array.from({ length: amountOfTables }, (_, index) => { + const tableName = `public.dummy_table_${index + 1}`; + const columns = [ + "id serial PRIMARY KEY", + "column_1 INTEGER NOT NULL", + "column_2 VARCHAR(100) NOT NULL", + "column_3 DECIMAL(10, 2) NOT NULL", + ]; + return createTable(tableName, columns); + }).join("\n"); + +export const dropDummyTablesQuery = (amountOfTables: number) => { + // postgres doesn't allow to drop multiple tables using wildcard, so need to compose the list of table names + const tables = Array.from({ length: amountOfTables }, (_, index) => `public.dummy_table_${index + 1}`).join(", "); + return dropTable(tables); +}; diff --git a/airbyte-webapp-e2e-tests/cypress/integration/connection/streamTable.spec.ts b/airbyte-webapp-e2e-tests/cypress/integration/connection/streamTable.spec.ts index f95a0361ae41..8b88d911469e 100644 --- a/airbyte-webapp-e2e-tests/cypress/integration/connection/streamTable.spec.ts +++ b/airbyte-webapp-e2e-tests/cypress/integration/connection/streamTable.spec.ts @@ -12,10 +12,15 @@ import { import { appendRandomString, submitButtonClick } from "commands/common"; import { clickNewConnectionButton, visitConnectionsListPage } from "pages/connnectionsListPage"; import { + checkAmountOfStreamTableRows, + checkColumnNames, + checkConnectorIconAndTitle, clickUseExistingConnectorButton, isAtConnectionOverviewPage, isAtNewConnectionPage, isNewConnectionPageHeaderVisible, + isStreamTableRowVisible, + scrollTableToStream, selectExistingConnectorFromDropdown, } from "pages/newConnectionPage"; import { @@ -29,12 +34,17 @@ import { waitForGetSourcesListRequest, } from "commands/interceptors"; import { Connection, Destination, Source } from "commands/api/types"; -import { selectSchedule } from "pages/replicationPage"; +import { clearStreamSearch, searchStream, selectSchedule } from "pages/replicationPage"; import { runDbQuery } from "commands/db/db"; -import { createUsersTableQuery, dropUsersTableQuery } from "commands/db/queries"; - -// TODO: Disable before merge -describe("New stream table - new connection set up ", () => { +import { + createUsersTableQuery, + dropUsersTableQuery, + createDummyTablesQuery, + dropDummyTablesQuery, +} from "commands/db/queries"; + +// TODO: Enable this test when the new stream table will be turned on +describe.skip("New stream table - new connection set up ", () => { let source: Source; let destination: Destination; let connectionId: string; @@ -42,7 +52,10 @@ describe("New stream table - new connection set up ", () => { before(() => { initialSetupCompleted(); runDbQuery(dropUsersTableQuery); + runDbQuery(dropDummyTablesQuery(20)); + runDbQuery(createUsersTableQuery); + runDbQuery(createDummyTablesQuery(20)); requestWorkspaceId().then(() => { const sourceRequestBody = getPostgresCreateSourceBody(appendRandomString("Stream table Source")); @@ -103,6 +116,37 @@ describe("New stream table - new connection set up ", () => { selectSchedule("Manual"); }); + it("should check check connector icons and titles in table", () => { + checkConnectorIconAndTitle("source"); + checkConnectorIconAndTitle("destination"); + }); + + it("should check columns names in table", () => { + checkColumnNames(); + }); + + it("should check total amount of table streams", () => { + // dummy tables amount + users table + checkAmountOfStreamTableRows(21); + }); + + it("should allow to scroll table to desired stream table row and it should be visible", () => { + const desiredStreamTableRow = "dummy_table_18"; + + scrollTableToStream(desiredStreamTableRow); + isStreamTableRowVisible(desiredStreamTableRow); + }); + + it("should filter table by stream name", () => { + searchStream("dummy_table_10"); + checkAmountOfStreamTableRows(1); + }); + + it("should clear stream search input field and show all available streams", () => { + clearStreamSearch(); + checkAmountOfStreamTableRows(21); + }); + /* here will be added more tests to extend the test flow */ diff --git a/airbyte-webapp-e2e-tests/cypress/pages/newConnectionPage.ts b/airbyte-webapp-e2e-tests/cypress/pages/newConnectionPage.ts index d2653cfb96d0..34c3209f7a41 100644 --- a/airbyte-webapp-e2e-tests/cypress/pages/newConnectionPage.ts +++ b/airbyte-webapp-e2e-tests/cypress/pages/newConnectionPage.ts @@ -7,6 +7,11 @@ const useExistingConnectorButton = (connectorType: ConnectorType) => const pageHeaderContainer = `div[data-testid='page-header-container']`; const newConnectionPageTitle = "New connection"; +const connectorHeaderGroupIcon = (connectorType: ConnectorType) => + `span[data-testid='connector-header-group-icon-container-${connectorType}']`; +const catalogTreeTableHeader = `div[data-testid='catalog-tree-table-header']`; +const catalogTreeTableBody = `div[data-testid='catalog-tree-table-body']`; + export const selectExistingConnectorFromDropdown = (connectorName: string) => cy .get(existingConnectorDropdown) @@ -19,7 +24,47 @@ export const clickUseExistingConnectorButton = (connectorType: ConnectorType) => export const isNewConnectionPageHeaderVisible = () => cy.get(pageHeaderContainer).contains(newConnectionPageTitle).should("be.visible"); -// Route checking +/* + Route checking + */ export const isAtNewConnectionPage = () => cy.url().should("include", `/connections/new-connection`); export const isAtConnectionOverviewPage = (connectionId: string) => cy.url().should("include", `connections/${connectionId}/status`); + +/* + Stream table + */ +export const checkConnectorIconAndTitle = (connectorType: ConnectorType) => { + const connectorIcon = connectorHeaderGroupIcon(connectorType); + cy.get(connectorIcon) + .contains(connectorType, { matchCase: false }) + .within(() => { + cy.get("img").should("have.attr", "src").should("not.be.empty"); + }); +}; + +export const checkColumnNames = () => { + const columnNames = ["Sync", "Namespace", "Stream name", "Sync mode", "Cursor field", "Primary key"]; + cy.get(catalogTreeTableHeader).within(($header) => { + columnNames.forEach((columnName) => { + cy.contains(columnName); + }); + // we have two Namespace columns + cy.get(`div:contains(${columnNames[1]})`).should("have.length", 2); + // we have two Stream Name columns + cy.get(`div:contains(${columnNames[2]})`).should("have.length", 2); + }); +}; + +export const checkAmountOfStreamTableRows = (expectedAmountOfRows: number) => + cy + .get(catalogTreeTableBody) + .find("[data-testid^='catalog-tree-table-row-']") + .should("have.length", expectedAmountOfRows); + +export const scrollTableToStream = (streamName: string) => { + cy.get(catalogTreeTableBody).contains(streamName).scrollIntoView(); +}; + +export const isStreamTableRowVisible = (streamName: string) => + cy.get(catalogTreeTableBody).contains(streamName).should("be.visible"); diff --git a/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts b/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts index fd4ac27f47de..3c55b5bf853f 100644 --- a/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts +++ b/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts @@ -178,6 +178,10 @@ export const searchStream = (value: string) => { cy.get(streamNameInput).type(value); }; +export const clearStreamSearch = () => { + cy.get(streamNameInput).clear(); +}; + export const clickSaveReplication = ({ reset = false, confirm = true } = {}) => { cy.intercept("/api/v1/web_backend/connections/update").as("updateConnection"); diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 9fe2df1b5708..31bcb5017117 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -70,6 +70,7 @@ "react-paginate": "^8.1.3", "react-query": "^3.39.1", "react-reflex": "^4.0.9", + "react-resize-detector": "^8.0.3", "react-router-dom": "6.3.0", "react-select": "^5.4.0", "react-slick": "^0.29.0", diff --git a/airbyte-webapp/pnpm-lock.yaml b/airbyte-webapp/pnpm-lock.yaml index e40571ff1720..2e4aa5b74d4c 100644 --- a/airbyte-webapp/pnpm-lock.yaml +++ b/airbyte-webapp/pnpm-lock.yaml @@ -114,6 +114,7 @@ specifiers: react-paginate: ^8.1.3 react-query: ^3.39.1 react-reflex: ^4.0.9 + react-resize-detector: ^8.0.3 react-router-dom: 6.3.0 react-select: ^5.4.0 react-select-event: ^5.5.0 @@ -197,6 +198,7 @@ dependencies: react-paginate: 8.1.4_react@17.0.2 react-query: 3.39.2_sfoxds7t5ydpegc3knd667wn6m react-reflex: 4.0.9_sfoxds7t5ydpegc3knd667wn6m + react-resize-detector: 8.0.3_sfoxds7t5ydpegc3knd667wn6m react-router-dom: 6.3.0_sfoxds7t5ydpegc3knd667wn6m react-select: 5.7.0_dlps62spiehie4hvtd46aaye5u react-slick: 0.29.0_sfoxds7t5ydpegc3knd667wn6m @@ -13083,6 +13085,17 @@ packages: react-dom: 17.0.2_react@17.0.2 dev: false + /react-resize-detector/8.0.3_sfoxds7t5ydpegc3knd667wn6m: + resolution: {integrity: sha512-c3eqm5BVcluVhxHsBQnhyPO/5uYB3XHIHz6D1ZOHzU2WcnZF0Cr3KLl5OIozRC2RSsdQlu5vn1PHEqrvKRnIYA==} + peerDependencies: + react: ^16.0.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 + dependencies: + lodash: 4.17.21 + react: 17.0.2 + react-dom: 17.0.2_react@17.0.2 + dev: false + /react-router-dom/6.3.0_sfoxds7t5ydpegc3knd667wn6m: resolution: {integrity: sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw==} peerDependencies: diff --git a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx index 1e1c4a051ec5..120e1a2e0890 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx +++ b/airbyte-webapp/src/components/ConnectorBlocks/FormPageContent.tsx @@ -1,6 +1,8 @@ import classNames from "classnames"; import { PropsWithChildren } from "react"; +import { FlexContainer } from "components/ui/Flex"; + import { isCloudApp } from "utils/app"; import styles from "./FormPageContent.module.scss"; @@ -10,14 +12,16 @@ interface FormPageContentProps { } const FormPageContent: React.FC> = ({ big, children }) => ( -
{children} -
+ ); export default FormPageContent; diff --git a/airbyte-webapp/src/components/connection/CatalogTree/CatalogTreeBody.tsx b/airbyte-webapp/src/components/connection/CatalogTree/CatalogTreeBody.tsx index c02d1e5c6a38..d24fe20957b8 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/CatalogTreeBody.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/CatalogTreeBody.tsx @@ -45,7 +45,7 @@ export const CatalogTreeBody: React.FC = ({ streams, chang ); return ( -
+
{isNewTableDesignEnabled ? ( <> diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableHeader.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableHeader.tsx index 64eedbbe235c..d511bbb78e0d 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableHeader.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/CatalogTreeTableHeader.tsx @@ -67,7 +67,10 @@ export const CatalogTreeTableHeader: React.FC = () => { }; return ( -
+
{mode !== "readonly" && ( = ({ const { streamHeaderContentStyle, pillButtonVariant } = useCatalogTreeTableRowProps(stream); return ( - + {!disabled && ( <> diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel/StreamFieldsTable/ConnectorHeaderGroupIcon.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel/StreamFieldsTable/ConnectorHeaderGroupIcon.tsx index af8983df82c9..7403660adf67 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel/StreamFieldsTable/ConnectorHeaderGroupIcon.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/StreamDetailsPanel/StreamFieldsTable/ConnectorHeaderGroupIcon.tsx @@ -12,7 +12,7 @@ interface StreamHeaderGroupIconProps { export const ConnectorHeaderGroupIcon: React.FC = ({ type, icon }) => { return ( - +
{getIcon(icon)}
 diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts index c60751213823..5c6c5863c606 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts @@ -500,6 +500,7 @@ describe("calculateInitialCatalog", () => { expect(calculatedStreams[0].stream?.defaultCursorField).toEqual(sourceDefinedStream?.defaultCursorField); expect(calculatedStreams[0].config?.cursorField).toEqual(calculatedStreams[0].stream?.defaultCursorField); }); + it("should keep original configured primary key if no source-defined primary key", () => { const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; @@ -531,6 +532,7 @@ describe("calculateInitialCatalog", () => { expect(calculatedStreams[0].config?.primaryKey).toEqual(config?.primaryKey); }); + it("should not override config cursor if sourceDefinedCursor is false", () => { const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; @@ -559,6 +561,7 @@ describe("calculateInitialCatalog", () => { expect(calculatedStreams[0].config?.cursorField).toEqual(config?.cursorField); }); + it("should keep its original config if source-defined primary key matches config primary key", () => { const { stream: sourceDefinedStream, config } = mockSyncSchemaStream; @@ -681,13 +684,12 @@ describe("calculateInitialCatalog", () => { const values = calculateInitialCatalog( { streams: [ + // Stream with breaking change { id: "1", stream: { ...sourceDefinedStream, name: "test", - sourceDefinedCursor: true, - defaultCursorField: ["id"], sourceDefinedPrimaryKey: [], supportedSyncModes: [SyncMode.incremental], }, @@ -698,13 +700,12 @@ describe("calculateInitialCatalog", () => { primaryKey: [["id"], ["email"]], }, }, + // Should not be affected { id: "2", stream: { ...sourceDefinedStream, name: "test-2", - sourceDefinedCursor: true, - defaultCursorField: ["updated_at"], sourceDefinedPrimaryKey: [], supportedSyncModes: [SyncMode.incremental], }, @@ -715,6 +716,22 @@ describe("calculateInitialCatalog", () => { primaryKey: [["id"]], }, }, + // Has change, but the source-defined primary key will fix it + { + id: "3", + stream: { + ...sourceDefinedStream, + name: "test-3", + sourceDefinedPrimaryKey: [["accountId"], ["userId"]], + supportedSyncModes: [SyncMode.incremental], + }, + config: { + ...config, + destinationSyncMode: DestinationSyncMode.append_dedup, + syncMode: SyncMode.incremental, + primaryKey: [["id"]], + }, + }, ], }, [DestinationSyncMode.append_dedup], @@ -730,11 +747,23 @@ describe("calculateInitialCatalog", () => { }, ], }, + { + transformType: StreamTransformTransformType.update_stream, + streamDescriptor: { name: "test-3", namespace: "namespace-test" }, + updateStream: [ + { + breaking: false, + transformType: FieldTransformTransformType.add_field, + fieldName: ["userId"], + }, + ], + }, ], true ); expect(values.streams[0].config?.primaryKey).toEqual([]); // was entirely cleared expect(values.streams[1].config?.primaryKey).toEqual([["id"]]); // was not affected + expect(values.streams[2].config?.primaryKey).toEqual([["accountId"], ["userId"]]); // was not affected because it's source-defined }); it("should remove cursor from config if the old cursor field was removed, even if there is a default", () => { @@ -742,6 +771,7 @@ describe("calculateInitialCatalog", () => { const values = calculateInitialCatalog( { streams: [ + // With breaking change { id: "1", stream: { @@ -749,7 +779,6 @@ describe("calculateInitialCatalog", () => { name: "test", sourceDefinedCursor: false, defaultCursorField: ["id"], - sourceDefinedPrimaryKey: [], supportedSyncModes: [SyncMode.incremental], }, config: { @@ -759,6 +788,7 @@ describe("calculateInitialCatalog", () => { cursorField: ["updated_at"], }, }, + // Will be unaffected { id: "2", stream: { @@ -776,6 +806,24 @@ describe("calculateInitialCatalog", () => { primaryKey: [["id"]], }, }, + // Has breaking change but the updated stream source-defined cursor will fix it + { + id: "3", + stream: { + ...sourceDefinedStream, + name: "test-3", + sourceDefinedCursor: true, + defaultCursorField: ["created_at"], + supportedSyncModes: [SyncMode.incremental], + }, + config: { + ...config, + destinationSyncMode: DestinationSyncMode.append_dedup, + syncMode: SyncMode.incremental, + cursorField: ["updated_at"], + primaryKey: [["id"]], + }, + }, ], }, [DestinationSyncMode.append_dedup], @@ -791,17 +839,36 @@ describe("calculateInitialCatalog", () => { }, ], }, + { + transformType: StreamTransformTransformType.update_stream, + streamDescriptor: { name: "test-3", namespace: "namespace-test" }, + updateStream: [ + { + breaking: true, + transformType: FieldTransformTransformType.remove_field, + fieldName: ["updated_at"], + }, + { + breaking: false, + transformType: FieldTransformTransformType.add_field, + fieldName: ["created_at"], + }, + ], + }, ], true ); expect(values.streams[0].config?.cursorField).toEqual([]); // was entirely cleared and not replaced with default expect(values.streams[1].config?.cursorField).toEqual(["updated_at"]); // was unaffected + expect(values.streams[2].config?.cursorField).toEqual(["created_at"]); // was unaffected }); + it("should clear multiple config fields if multiple fields were removed", () => { const { config, stream: sourceDefinedStream } = mockSyncSchemaStream; const values = calculateInitialCatalog( { streams: [ + // Breaking. Should be cleared { id: "1", stream: { @@ -820,6 +887,7 @@ describe("calculateInitialCatalog", () => { primaryKey: [["primary_key"], ["another_field"]], }, }, + // Should be unaffected { id: "2", stream: { @@ -838,6 +906,25 @@ describe("calculateInitialCatalog", () => { primaryKey: [["id"]], }, }, + // Should stay unaffected because updated stream will assign new source-defined cursor + { + id: "3", + stream: { + ...sourceDefinedStream, + name: "test-3", + sourceDefinedCursor: true, + defaultCursorField: ["created_at"], + sourceDefinedPrimaryKey: [], + supportedSyncModes: [SyncMode.incremental], + }, + config: { + ...config, + destinationSyncMode: DestinationSyncMode.append_dedup, + syncMode: SyncMode.incremental, + cursorField: ["updated_at"], + primaryKey: [["id"]], + }, + }, ], }, [DestinationSyncMode.append_dedup], @@ -858,13 +945,38 @@ describe("calculateInitialCatalog", () => { }, ], }, + { + transformType: StreamTransformTransformType.update_stream, + streamDescriptor: { name: "test-3", namespace: "namespace-test" }, + updateStream: [ + { + breaking: true, + transformType: FieldTransformTransformType.remove_field, + fieldName: ["updated_at"], + }, + { + breaking: false, + transformType: FieldTransformTransformType.add_field, + fieldName: ["created_at"], + }, + { + breaking: true, + transformType: FieldTransformTransformType.remove_field, + fieldName: ["primary_key"], + }, + ], + }, ], true ); + expect(values.streams[0].config?.primaryKey).toEqual([]); // was entirely cleared and not replaced with default expect(values.streams[0].config?.cursorField).toEqual([]); // was entirely cleared and not replaced with default - expect(values.streams[1].config?.primaryKey).toEqual([["id"]]); // was unaffected}) - expect(values.streams[1].config?.cursorField).toEqual(["updated_at"]); // was unaffected}) + expect(values.streams[1].config?.primaryKey).toEqual([["id"]]); // was unaffected + expect(values.streams[1].config?.cursorField).toEqual(["updated_at"]); // was unaffected + + expect(values.streams[2].config?.primaryKey).toEqual([["id"]]); // was unaffected + expect(values.streams[2].config?.cursorField).toEqual(["created_at"]); // was unaffected because it's a source-defined cursor }); }); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts index f0db54cc38e9..98e3fe50e2f2 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts @@ -21,42 +21,47 @@ const clearBreakingFieldChanges = (nodeStream: SyncSchemaStream, breakingChanges return nodeStream; } + const { primaryKey, cursorField } = nodeStream.config; + let clearPrimaryKey = false; let clearCursorField = false; for (const streamTransformation of breakingChangesByStream) { - // get all of the removed field paths for this transformation - const removedFieldPaths = streamTransformation.updateStream?.map((update) => update.fieldName); - - if (!removedFieldPaths?.length) { + if (!streamTransformation.updateStream || !streamTransformation.updateStream?.length) { continue; } + // get all of the removed field paths for this transformation + const breakingFieldPaths = streamTransformation.updateStream + .filter(({ breaking }) => breaking) + .map((update) => update.fieldName); + // if there is a primary key in the config, and any of its field paths were removed, we'll be clearing it if ( - !!nodeStream.config?.primaryKey?.length && - nodeStream.config?.primaryKey?.some((key) => removedFieldPaths.some((removedPath) => isEqual(key, removedPath))) + !!primaryKey?.length && + primaryKey?.some((primaryKeyPath) => breakingFieldPaths.some((path) => isEqual(primaryKeyPath, path))) ) { clearPrimaryKey = true; } // if there is a cursor field, and any of its field path was removed, we'll be clearing it - if ( - !!nodeStream.config?.cursorField?.length && - removedFieldPaths.some((removedPath) => isEqual(removedPath, nodeStream?.config?.cursorField)) - ) { + if (!!cursorField?.length && breakingFieldPaths.some((path) => isEqual(path, cursorField))) { clearCursorField = true; } } - return { - ...nodeStream, - config: { - ...nodeStream.config, - primaryKey: clearPrimaryKey ? [] : nodeStream.config.primaryKey, - cursorField: clearCursorField ? [] : nodeStream.config.cursorField, - }, - }; + if (clearPrimaryKey || clearCursorField) { + return { + ...nodeStream, + config: { + ...nodeStream.config, + primaryKey: clearPrimaryKey ? [] : nodeStream.config.primaryKey, + cursorField: clearCursorField ? [] : nodeStream.config.cursorField, + }, + }; + } + + return nodeStream; }; const verifySourceDefinedProperties = (streamNode: SyncSchemaStream, isEditMode: boolean) => { @@ -187,19 +192,19 @@ const calculateInitialCatalog = ( streamIdFromDiff.namespace === nodeStream.stream?.namespace ); - // narrow down the breaking field changes from this connection to only those relevant to this stream - const breakingChangesByStream = - streamsWithBreakingFieldChanges && streamsWithBreakingFieldChanges.length > 0 - ? streamsWithBreakingFieldChanges.filter((streamTransformFromDiff) => { - return ( - streamTransformFromDiff.streamDescriptor.name === nodeStream.stream?.name && - streamTransformFromDiff.streamDescriptor.namespace === nodeStream.stream?.namespace - ); - }) - : []; - // if we're in edit or readonly mode and the stream is not new, check for breaking changes then return if (isNotCreateMode && !isStreamNew) { + // narrow down the breaking field changes from this connection to only those relevant to this stream + const breakingChangesByStream = + streamsWithBreakingFieldChanges && streamsWithBreakingFieldChanges.length > 0 + ? streamsWithBreakingFieldChanges.filter(({ streamDescriptor }) => { + return ( + streamDescriptor.name === nodeStream.stream?.name && + streamDescriptor.namespace === nodeStream.stream?.namespace + ); + }) + : []; + return clearBreakingFieldChanges(nodeStream, breakingChangesByStream); } diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap index 81516c7444e2..d612bc30ae37 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap @@ -605,6 +605,7 @@ exports[`CreateConnectionForm should render 1`] = ` >
= ({ builderRoutePath }) => { + const navigate = useNavigate(); + const { width, ref } = useResizeDetector(); + const applyNarrowLayout = Boolean(width && width < 460); + + return ( + + + + + + + + + + + + ), + noun: ( + + + + ), + }} + /> + + + + + + ); +}; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx index 3db5f13352f2..42f573058235 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/PageDisplay.tsx @@ -1,11 +1,12 @@ import { Tab } from "@headlessui/react"; import classNames from "classnames"; import { useField } from "formik"; -import React, { useMemo } from "react"; -import { useIntl } from "react-intl"; +import React, { ReactNode, useMemo } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; +import { InfoTooltip } from "components/ui/Tooltip"; import { StreamReadInferredSchema, StreamReadSlicesItemPagesItem } from "core/request/ConnectorBuilderClient"; import { @@ -25,7 +26,7 @@ interface PageDisplayProps { } interface TabData { - title: string; + title: ReactNode; key: string; content: string; } @@ -34,7 +35,7 @@ export const PageDisplay: React.FC = ({ page, className, infer const { formatMessage } = useIntl(); const { editorView } = useConnectorBuilderFormState(); - const { testStreamIndex } = useConnectorBuilderTestState(); + const { testStreamIndex, streamRead } = useConnectorBuilderTestState(); const [field] = useField(`streams[${testStreamIndex}].schema`); const formattedRecords = useMemo(() => formatJson(page.records), [page.records]); @@ -45,7 +46,16 @@ export const PageDisplay: React.FC = ({ page, className, infer let defaultTabIndex = 0; const tabs: TabData[] = [ { - title: `${formatMessage({ id: "connectorBuilder.recordsTab" })} (${page.records.length})`, + title: ( + <> + {`${formatMessage({ id: "connectorBuilder.recordsTab" })} (${page.records.length})`} + {!streamRead.isFetching && streamRead.data && streamRead.data.test_read_limit_reached && ( + + + + )} + + ), key: "records", content: formattedRecords, }, diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx index cdfdc04a132b..7312a87e36ba 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx @@ -1,6 +1,12 @@ +import { faClose } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { useEffect, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; +import { useLocalStorage } from "react-use"; +import { Button } from "components/ui/Button"; +import { Callout } from "components/ui/Callout"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; import { ResizablePanels } from "components/ui/ResizablePanels"; import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; @@ -37,6 +43,7 @@ export const StreamTester: React.FC<{ errorUpdatedAt, }, } = useConnectorBuilderTestState(); + const [showLimitWarning, setShowLimitWarning] = useLocalStorage("connectorBuilderLimitWarning", true); const streamName = streams[testStreamIndex]?.name; @@ -139,6 +146,24 @@ export const StreamTester: React.FC<{
)} + {!isFetching && streamReadData && streamReadData.test_read_limit_reached && showLimitWarning && ( + + + + + + +
); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx index 265bc4d08aa8..8a6c092cb987 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx @@ -1,3 +1,4 @@ +import { faDocker } from "@fortawesome/free-brands-svg-icons"; import { faPlus } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { useState } from "react"; @@ -5,16 +6,19 @@ import { FormattedMessage, useIntl } from "react-intl"; import { useNavigate } from "react-router-dom"; import { Button } from "components/ui/Button"; +import { DropdownMenu, DropdownMenuOptionType } from "components/ui/DropdownMenu"; +import { useExperiment } from "hooks/services/Experiment"; import { RoutePaths, DestinationPaths } from "pages/routePaths"; import { useCreateDestinationDefinition } from "services/connector/DestinationDefinitionService"; import { useCreateSourceDefinition } from "services/connector/SourceDefinitionService"; import { useCurrentWorkspaceId } from "services/workspaces/WorkspacesService"; +import { ReactComponent as BuilderIcon } from "./builder-icon.svg"; import CreateConnectorModal from "./CreateConnectorModal"; interface IProps { - type: string; + type: "sources" | "destinations"; } interface ICreateProps { @@ -33,6 +37,7 @@ const CreateConnector: React.FC = ({ type }) => { setIsModalOpen(!isModalOpen); setErrorMessage(""); }; + const showBuilderNavigationLinks = useExperiment("connectorBuilder.showNavigationLinks", false); const { formatMessage } = useIntl(); @@ -77,10 +82,30 @@ const CreateConnector: React.FC = ({ type }) => { return ( <> - {type === "configuration" ? null : ( - + {type === "sources" && showBuilderNavigationLinks ? ( + , + displayName: formatMessage({ id: "admin.newConnector.build" }), + internal: true, + }, + { + as: "button", + icon: , + value: "docker", + displayName: formatMessage({ id: "admin.newConnector.docker" }), + }, + ]} + onChange={(data: DropdownMenuOptionType) => data.value === "docker" && onChangeModalState()} + > + {() => } + + ) : ( + )} {isModalOpen && ( @@ -90,4 +115,16 @@ const CreateConnector: React.FC = ({ type }) => { ); }; +interface NewConnectorButtonProps { + onClick?: () => void; +} + +const NewConnectorButton: React.FC = ({ onClick }) => { + return ( + + ); +}; + export default CreateConnector; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.module.scss index 9cdcb2d1f495..f095e13eaebb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.module.scss +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.module.scss @@ -18,7 +18,7 @@ position: relative; background: colors.$white; - &::before { + &::after { position: absolute; display: block; pointer-events: none; @@ -26,10 +26,9 @@ color: colors.$grey-400; top: 10px; right: 22px; - z-index: 3; } - &:focus-within::before { + &:focus-within::after { display: none; } } diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/builder-icon.svg b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/builder-icon.svg new file mode 100644 index 000000000000..15cbec80e627 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/builder-icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/SourceForm.module.scss b/airbyte-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/SourceForm.module.scss new file mode 100644 index 000000000000..d7d67460922c --- /dev/null +++ b/airbyte-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/SourceForm.module.scss @@ -0,0 +1,6 @@ +@use "scss/variables"; + +.builderPrompt { + padding: variables.$spacing-xl; + align-self: center; +} diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/SourceForm.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/SourceForm.tsx index 136f64426500..c5ae39f24b53 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/SourceForm.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/SourceForm.tsx @@ -2,14 +2,21 @@ import React, { useState } from "react"; import { FormattedMessage } from "react-intl"; import { useLocation } from "react-router-dom"; +import { BuilderPrompt } from "components/connectorBuilder/BuilderPrompt"; +import { Card } from "components/ui/Card"; + import { ConnectionConfiguration } from "core/domain/connection"; import { LogsRequestError } from "core/request/LogsRequestError"; +import { useExperiment } from "hooks/services/Experiment"; +import { RoutePaths } from "pages/routePaths"; import { SourceDefinitionReadWithLatestTag } from "services/connector/SourceDefinitionService"; import { useGetSourceDefinitionSpecificationAsync } from "services/connector/SourceDefinitionSpecificationService"; import { FormError } from "utils/errorStatusMessage"; import { ConnectorCard } from "views/Connector/ConnectorCard"; import { ConnectorCardValues } from "views/Connector/ConnectorForm/types"; +import styles from "./SourceForm.module.scss"; + interface SourceFormProps { onSubmit: (values: { name: string; @@ -53,19 +60,28 @@ export const SourceForm: React.FC = ({ onSubmit, sourceDefiniti }); }; + const showBuilderNavigationLinks = useExperiment("connectorBuilder.showNavigationLinks", false); + return ( - } - description={} - isLoading={isLoading} - fetchingConnectorError={sourceDefinitionError instanceof Error ? sourceDefinitionError : null} - availableConnectorDefinitions={sourceDefinitions} - onConnectorDefinitionSelect={onDropDownSelect} - selectedConnectorDefinitionSpecification={sourceDefinitionSpecification} - selectedConnectorDefinitionId={sourceDefinitionId} - onSubmit={onSubmitForm} - jobInfo={LogsRequestError.extractJobInfo(error)} - /> + <> + } + description={} + isLoading={isLoading} + fetchingConnectorError={sourceDefinitionError instanceof Error ? sourceDefinitionError : null} + availableConnectorDefinitions={sourceDefinitions} + onConnectorDefinitionSelect={onDropDownSelect} + selectedConnectorDefinitionSpecification={sourceDefinitionSpecification} + selectedConnectorDefinitionId={sourceDefinitionId} + onSubmit={onSubmitForm} + jobInfo={LogsRequestError.extractJobInfo(error)} + /> + {showBuilderNavigationLinks && !sourceDefinitionSpecification && ( + + + + )} + ); }; diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap index bddc7311ba78..e5bc00f72d92 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/__snapshots__/ConnectionReplicationPage.test.tsx.snap @@ -536,6 +536,7 @@ exports[`ConnectionReplicationPage should render 1`] = ` >
void; } -const FormContent = styled(Form)` - padding: 22px 27px 23px 24px; -`; - -const BottomBlock = styled.div` - text-align: right; - margin-top: 34px; -`; - -const PaddingBlock = styled.div` - text-align: center; - padding: 18px 0 15px; - font-weight: 500; - font-size: 15px; - line-height: 18px; -`; - const existingEntityValidationSchema = yup.object().shape({ entityId: yup.string().required("form.empty.error"), }); @@ -94,7 +77,7 @@ const ExistingEntityForm: React.FC = ({ type, onSubmit }) => { }} > {({ isSubmitting, setFieldValue }) => ( - +
{({ field }: FieldProps) => ( = ({ type, onSubmit }) => { )} - - - - + +
)} - + - + ); }; diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderEditPage/ConnectorBuilderEditPage.tsx b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderEditPage/ConnectorBuilderEditPage.tsx index 9bffb1d33be7..de4dbaaa1977 100644 --- a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderEditPage/ConnectorBuilderEditPage.tsx +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderEditPage/ConnectorBuilderEditPage.tsx @@ -3,6 +3,7 @@ import { Formik } from "formik"; import React, { useCallback, useEffect, useMemo, useRef } from "react"; import { useIntl } from "react-intl"; +import { HeadTitle } from "components/common/HeadTitle"; import { Builder } from "components/connectorBuilder/Builder/Builder"; import { StreamTestingPanel } from "components/connectorBuilder/StreamTestingPanel"; import { builderFormValidationSchema, BuilderFormValues } from "components/connectorBuilder/types"; @@ -67,6 +68,7 @@ export const ConnectorBuilderEditPage: React.FC = () => ( + @@ -102,7 +104,7 @@ const Panels = React.memo( ), className: styles.leftPanel, - minWidth: 100, + minWidth: 550, }} secondPanel={{ children: , diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/ConnectorBuilderLandingPage.tsx b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/ConnectorBuilderLandingPage.tsx index b0a228ee4c1f..253a0d3228d1 100644 --- a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/ConnectorBuilderLandingPage.tsx +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/ConnectorBuilderLandingPage.tsx @@ -8,6 +8,7 @@ import React, { useCallback, useEffect, useRef, useState } from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; +import { HeadTitle } from "components/common/HeadTitle"; import { BuilderFormValues, DEFAULT_BUILDER_FORM_VALUES, @@ -30,10 +31,10 @@ import { useConnectorBuilderLocalStorage, } from "services/connectorBuilder/ConnectorBuilderLocalStorageService"; +import { ReactComponent as AirbyteLogo } from "./airbyte-logo.svg"; import styles from "./ConnectorBuilderLandingPage.module.scss"; -import { ReactComponent as AirbyteLogo } from "../../../../public/images/airbyte/logo.svg"; -import { ReactComponent as ImportYamlImage } from "../../../../public/images/connector-builder/import-yaml.svg"; -import { ReactComponent as StartFromScratchImage } from "../../../../public/images/connector-builder/start-from-scratch.svg"; +import { ReactComponent as ImportYamlImage } from "./import-yaml.svg"; +import { ReactComponent as StartFromScratchImage } from "./start-from-scratch.svg"; import { ConnectorBuilderRoutePaths } from "../ConnectorBuilderRoutes"; const YAML_UPLOAD_ERROR_ID = "connectorBuilder.yamlUpload.error"; @@ -210,6 +211,7 @@ const ConnectorBuilderLandingPageInner: React.FC = () => { export const ConnectorBuilderLandingPage: React.FC = () => ( + ); diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/airbyte-logo.svg b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/airbyte-logo.svg new file mode 100644 index 000000000000..a2f78bbe3972 --- /dev/null +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/airbyte-logo.svg @@ -0,0 +1,4 @@ + + + + diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/import-yaml.svg b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/import-yaml.svg new file mode 100644 index 000000000000..62b89447dcec --- /dev/null +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/import-yaml.svg @@ -0,0 +1,155 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/load-existing-connector.svg b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/load-existing-connector.svg new file mode 100644 index 000000000000..c322290d539f --- /dev/null +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/load-existing-connector.svg @@ -0,0 +1,352 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/start-from-scratch.svg b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/start-from-scratch.svg new file mode 100644 index 000000000000..7b152f391e4e --- /dev/null +++ b/airbyte-webapp/src/pages/connectorBuilder/ConnectorBuilderLandingPage/start-from-scratch.svg @@ -0,0 +1,223 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 4eb51539070c..8b7ffa45910d 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -56,6 +56,7 @@ const MainViewRoutes: React.FC = () => { } /> } /> } /> + } /> } /> @@ -105,11 +106,6 @@ export const Routing: React.FC = () => { ); return ( - } />} - /> - } /> {OldRoutes} } /> } /> diff --git a/airbyte-webapp/src/scss/_theme-light.scss b/airbyte-webapp/src/scss/_theme-light.scss index 0004ebfcfa97..7dc9e6d9c246 100644 --- a/airbyte-webapp/src/scss/_theme-light.scss +++ b/airbyte-webapp/src/scss/_theme-light.scss @@ -91,9 +91,9 @@ --color-overlay-background: rgba(26 25 77 / 50%); --box-shadow: 0 2px 4px rgba(26 25 77 / 12%); - --box-shadow-left: -2px 0 10px rgba(26 25 77, 12%); - --box-shadow-right: 2px 0 10px rgba(26 25 77, 12%); + --box-shadow-left: -2px 0 10px rgba(26 25 77 / 12%); + --box-shadow-right: 2px 0 10px rgba(26 25 77 / 12%); --box-shadow-raised: 0 10px 19px rgba(26 25 77 / 16%); - --box-shadow-popup: 0 0 22px rgba(26 25 77, 12%); + --box-shadow-popup: 0 0 22px rgba(26 25 77 / 12%); --box-shadow-sidebar: 0 2px 4px rgba(26 25 77 / 5%); } diff --git a/airbyte-webapp/src/utils/links.ts b/airbyte-webapp/src/utils/links.ts index e36d3f1a9559..ddbdb83ecb62 100644 --- a/airbyte-webapp/src/utils/links.ts +++ b/airbyte-webapp/src/utils/links.ts @@ -6,6 +6,7 @@ const BASE_DOCS_LINK = "https://docs.airbyte.com"; export const links = { dbtCommandsReference: "https://docs.getdbt.com/reference/dbt-commands", + dbtCloudIntegrationDocs: `${BASE_DOCS_LINK}/cloud/dbt-cloud-integration`, technicalSupport: `${BASE_DOCS_LINK}/troubleshooting/on-deploying`, termsLink: "https://airbyte.com/terms", privacyLink: "https://airbyte.com/privacy-policy", diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx index c8f1bad0f692..caf4bbbde434 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx @@ -227,33 +227,35 @@ export const ConnectorCard: React.FC ( - { - if (!selectedConnectorDefinitionId) { - return; - } - handleTestConnector( - isEditMode ? undefined : { ...getValues(), serviceType: selectedConnectorDefinitionId } - ); - }} - onDeleteClick={onDeleteClick} - isValid={isValid} - dirty={dirty} - job={job ? job : undefined} - onCancelClick={() => { - resetConnectorForm(); - }} - connectionTestSuccess={connectionTestSuccess} - /> - )} + renderFooter={({ dirty, isSubmitting, isValid, resetConnectorForm, getValues }) => + selectedConnectorDefinitionSpecification && ( + { + if (!selectedConnectorDefinitionId) { + return; + } + handleTestConnector( + isEditMode ? undefined : { ...getValues(), serviceType: selectedConnectorDefinitionId } + ); + }} + onDeleteClick={onDeleteClick} + isValid={isValid} + dirty={dirty} + job={job ? job : undefined} + onCancelClick={() => { + resetConnectorForm(); + }} + connectionTestSuccess={connectionTestSuccess} + /> + ) + } renderWithCard /> ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Controls/ConnectorServiceTypeControl/ConnectorDefinitionTypeControl.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Controls/ConnectorServiceTypeControl/ConnectorDefinitionTypeControl.tsx index 2286fe14d5c7..4a9fddc9dba7 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Controls/ConnectorServiceTypeControl/ConnectorDefinitionTypeControl.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Controls/ConnectorServiceTypeControl/ConnectorDefinitionTypeControl.tsx @@ -5,6 +5,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import { components } from "react-select"; import { MenuListProps } from "react-select"; +import { BuilderPrompt } from "components/connectorBuilder/BuilderPrompt"; import { GAIcon } from "components/icons/GAIcon"; import { ControlLabels } from "components/LabeledControl"; import { @@ -26,6 +27,7 @@ import { useFeature, FeatureItem } from "hooks/services/Feature"; import { useModalService } from "hooks/services/Modal"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import { FreeTag } from "packages/cloud/components/experiments/FreeConnectorProgram"; +import { RoutePaths } from "pages/routePaths"; import { useDocumentationPanelContext } from "views/Connector/ConnectorDocumentationLayout/DocumentationPanelContext"; import RequestConnectorModal from "views/Connector/RequestConnectorModal"; @@ -37,20 +39,29 @@ import { WarningMessage } from "../../WarningMessage"; // eslint-disable-next-line @typescript-eslint/no-explicit-any type MenuWithRequestButtonProps = MenuListProps & { selectProps: any }; -const ConnectorList: React.FC> = ({ children, ...props }) => ( - <> - {children} -
- -
- -); +const ConnectorList: React.FC> = ({ children, ...props }) => { + const showBuilderNavigationLinks = useExperiment("connectorBuilder.showNavigationLinks", false); + + return ( + <> + {children} +
+ {props.selectProps.selectProps.formType === "source" && showBuilderNavigationLinks && ( +
+ +
+ )} + +
+ + ); +}; const StageLabel: React.FC<{ releaseStage?: ReleaseStage }> = ({ releaseStage }) => { const fcpEnabled = useFeature(FeatureItem.FreeConnectorProgram); @@ -173,6 +184,7 @@ export const ConnectorDefinitionTypeControl: React.FC ), }), + formType, }), [closeModal, formType, formatMessage, openModal, workspace.email] ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Controls/ConnectorServiceTypeControl/ConnectorServiceTypeControl.module.scss b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Controls/ConnectorServiceTypeControl/ConnectorServiceTypeControl.module.scss index 5cb031bfb35e..3729c020398b 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Controls/ConnectorServiceTypeControl/ConnectorServiceTypeControl.module.scss +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Controls/ConnectorServiceTypeControl/ConnectorServiceTypeControl.module.scss @@ -7,6 +7,9 @@ border-top: variables.$border-thin solid colors.$grey-50; min-height: 36px; position: relative; + display: flex; + flex-direction: column; + gap: variables.$spacing-md; .requestNewConnectorBtn { display: flex; @@ -16,7 +19,7 @@ border: none; cursor: pointer; background: colors.$white; - color: colors.$dark-blue; + color: colors.$blue; &:hover { color: colors.$blue; @@ -51,3 +54,9 @@ justify-content: space-between; align-items: center; } + +.builderPromptContainer { + border: 1px solid colors.$grey-100; + border-radius: variables.$border-radius-md; + padding: variables.$spacing-md; +} diff --git a/airbyte-webapp/src/views/layout/SideBar/SideBar.tsx b/airbyte-webapp/src/views/layout/SideBar/SideBar.tsx index c4258aabf31f..219c8684394f 100644 --- a/airbyte-webapp/src/views/layout/SideBar/SideBar.tsx +++ b/airbyte-webapp/src/views/layout/SideBar/SideBar.tsx @@ -11,9 +11,11 @@ import { Version } from "components/common/Version"; import { Text } from "components/ui/Text"; import { useConfig } from "config"; +import { useExperiment } from "hooks/services/Experiment"; import { links } from "utils/links"; import { ReactComponent as AirbyteLogo } from "./airbyteLogo.svg"; +import BuilderIcon from "./components/BuilderIcon"; import ConnectionsIcon from "./components/ConnectionsIcon"; import DestinationIcon from "./components/DestinationIcon"; import RecipesIcon from "./components/RecipesIcon"; @@ -40,6 +42,7 @@ const SideBar: React.FC = () => { const config = useConfig(); const navLinkClassName = useCalculateSidebarStyles(); const { formatMessage } = useIntl(); + const showBuilderNavigationLinks = useExperiment("connectorBuilder.showNavigationLinks", false); return (
    diff --git a/airbyte-webapp/src/views/layout/SideBar/components/BuilderIcon.tsx b/airbyte-webapp/src/views/layout/SideBar/components/BuilderIcon.tsx new file mode 100644 index 000000000000..22d450432cfd --- /dev/null +++ b/airbyte-webapp/src/views/layout/SideBar/components/BuilderIcon.tsx @@ -0,0 +1,10 @@ +const BuilderIcon = ({ color = "currentColor" }: { color?: string }): JSX.Element => ( + + + +); + +export default BuilderIcon; diff --git a/airbyte-webapp/vite.config.ts b/airbyte-webapp/vite.config.ts index 6609883cb9f2..4dafa3384dbb 100644 --- a/airbyte-webapp/vite.config.ts +++ b/airbyte-webapp/vite.config.ts @@ -43,7 +43,7 @@ export default defineConfig(({ mode }) => { // Align error popover button with the react-query dev tool button badgeStyle: "transform: translate(-135px,-11px)", }, - eslint: { lintCommand: `eslint --max-warnings=0 --ext js,ts,tsx src` }, + eslint: { lintCommand: `eslint --max-warnings=0 --ext .js,.ts,.tsx src` }, stylelint: { lintCommand: 'stylelint "src/**/*.{css,scss}"', // We need to overwrite this during development, since otherwise `files` are wrongly diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 4f8416a939ea..984553e55a34 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -67,7 +67,7 @@ dependencies { implementation project(':airbyte-featureflag') implementation project(':airbyte-metrics:metrics-lib') implementation project(':airbyte-json-validation') - implementation project(':airbyte-protocol:protocol-models') + implementation libs.airbyte.protocol implementation project(':airbyte-notification') implementation(project(':airbyte-persistence:job-persistence')) { // Temporary hack to avoid dependency conflicts diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionNotificationWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionNotificationWorkflowImpl.java index 842154263deb..ca3d59a69cec 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionNotificationWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionNotificationWorkflowImpl.java @@ -6,11 +6,8 @@ import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.temporal.scheduling.ConnectionNotificationWorkflow; -import io.airbyte.config.Notification; -import io.airbyte.config.Notification.NotificationType; import io.airbyte.config.SlackNotificationConfiguration; import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.notification.SlackNotificationClient; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.annotations.TemporalActivityStub; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity; @@ -36,7 +33,7 @@ public class ConnectionNotificationWorkflowImpl implements ConnectionNotificatio private ConfigFetchActivity configFetchActivity; @Override - public boolean sendSchemaChangeNotification(final UUID connectionId) + public boolean sendSchemaChangeNotification(final UUID connectionId, final String url) throws IOException, InterruptedException, ApiException, ConfigNotFoundException, JsonValidationException { final int getBreakingChangeVersion = Workflow.getVersion(GET_BREAKING_CHANGE_TAG, Workflow.DEFAULT_VERSION, GET_BREAKING_CHANGE_VERSION); @@ -44,11 +41,7 @@ public boolean sendSchemaChangeNotification(final UUID connectionId) final Optional breakingChange = configFetchActivity.getBreakingChange(connectionId); final Optional slackConfig = slackConfigActivity.fetchSlackConfiguration(connectionId); if (slackConfig.isPresent() && breakingChange.isPresent()) { - final Notification notification = - new Notification().withNotificationType(NotificationType.SLACK).withSendOnFailure(false).withSendOnSuccess(false) - .withSlackConfiguration(slackConfig.get()); - final SlackNotificationClient notificationClient = new SlackNotificationClient(notification); - return notifySchemaChangeActivity.notifySchemaChange(notificationClient, connectionId, breakingChange.get()); + return notifySchemaChangeActivity.notifySchemaChange(connectionId, breakingChange.get(), slackConfig.get(), url); } else { return false; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivity.java index 24e9c62661fa..3fd35e0aeee9 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivity.java @@ -4,7 +4,7 @@ package io.airbyte.workers.temporal.scheduling.activities; -import io.airbyte.notification.SlackNotificationClient; +import io.airbyte.config.SlackNotificationConfiguration; import io.temporal.activity.ActivityInterface; import io.temporal.activity.ActivityMethod; import java.io.IOException; @@ -14,7 +14,7 @@ public interface NotifySchemaChangeActivity { @ActivityMethod - public boolean notifySchemaChange(SlackNotificationClient notificationClient, UUID connectionId, boolean isBreaking) + public boolean notifySchemaChange(UUID connectionId, boolean isBreaking, SlackNotificationConfiguration config, String url) throws IOException, InterruptedException; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivityImpl.java index bc5707bdf0ae..fdd012ec0c18 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivityImpl.java @@ -4,6 +4,9 @@ package io.airbyte.workers.temporal.scheduling.activities; +import io.airbyte.config.Notification; +import io.airbyte.config.Notification.NotificationType; +import io.airbyte.config.SlackNotificationConfiguration; import io.airbyte.notification.SlackNotificationClient; import jakarta.inject.Singleton; import java.io.IOException; @@ -13,9 +16,20 @@ public class NotifySchemaChangeActivityImpl implements NotifySchemaChangeActivity { @Override - public boolean notifySchemaChange(SlackNotificationClient notificationClient, UUID connectionId, boolean isBreaking) + public boolean notifySchemaChange(UUID connectionId, boolean isBreaking, SlackNotificationConfiguration slackConfig, String url) throws IOException, InterruptedException { - return notificationClient.notifySchemaChange(connectionId, isBreaking); + final Notification notification = createNotification(slackConfig); + final SlackNotificationClient notificationClient = createNotificationClient(notification); + return notificationClient.notifySchemaChange(connectionId, isBreaking, slackConfig, url); + } + + Notification createNotification(SlackNotificationConfiguration slackConfig) { + return new Notification().withNotificationType(NotificationType.SLACK).withSendOnFailure(false).withSendOnSuccess(false) + .withSlackConfiguration(slackConfig); + } + + SlackNotificationClient createNotificationClient(Notification notification) { + return new SlackNotificationClient(notification); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index f6731d438c43..dc30237d498c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -56,7 +56,7 @@ public void refreshSchema(final UUID sourceCatalogId, final UUID connectionId) { ApmTraceUtils.addTagsToTrace(Map.of(CONNECTION_ID_KEY, connectionId, SOURCE_ID_KEY, sourceCatalogId)); final SourceDiscoverSchemaRequestBody requestBody = - new SourceDiscoverSchemaRequestBody().sourceId(sourceCatalogId).disableCache(true).connectionId(connectionId); + new SourceDiscoverSchemaRequestBody().sourceId(sourceCatalogId).disableCache(true).connectionId(connectionId).notifySchemaChange(true); try { sourceApi.discoverSchemaForSource(requestBody); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionNotificationWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionNotificationWorkflowTest.java index 65d554e760c7..9687f584a4c3 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionNotificationWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionNotificationWorkflowTest.java @@ -14,7 +14,6 @@ import io.airbyte.commons.temporal.scheduling.ConnectionNotificationWorkflow; import io.airbyte.config.SlackNotificationConfiguration; import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.notification.SlackNotificationClient; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivityImpl; import io.airbyte.workers.temporal.scheduling.activities.NotifySchemaChangeActivityImpl; @@ -67,7 +66,9 @@ void setUp() throws IOException, InterruptedException, ApiException, JsonValidat .build(); mNotifySchemaChangeActivity = mock(NotifySchemaChangeActivityImpl.class); - when(mNotifySchemaChangeActivity.notifySchemaChange(any(SlackNotificationClient.class), any(UUID.class), any(boolean.class))).thenReturn(true); + when(mNotifySchemaChangeActivity.notifySchemaChange(any(UUID.class), any(boolean.class), any(SlackNotificationConfiguration.class), + any(String.class))) + .thenReturn(true); mSlackConfigActivity = mock(SlackConfigActivityImpl.class); when(mSlackConfigActivity.fetchSlackConfiguration(any(UUID.class))).thenReturn( @@ -100,11 +101,13 @@ void sendSchemaChangeNotificationNonBreakingChangeTest() client.newWorkflowStub(ConnectionNotificationWorkflow.class, WorkflowOptions.newBuilder().setTaskQueue(NOTIFICATIONS_QUEUE).build()); final UUID connectionId = UUID.randomUUID(); + final String connectionUrl = "connection_url"; when(mConfigFetchActivity.getBreakingChange(connectionId)).thenReturn(Optional.of(false)); - workflow.sendSchemaChangeNotification(connectionId); + workflow.sendSchemaChangeNotification(connectionId, connectionUrl); - verify(mNotifySchemaChangeActivity, times(1)).notifySchemaChange(any(SlackNotificationClient.class), any(UUID.class), any(boolean.class)); + verify(mNotifySchemaChangeActivity, times(1)).notifySchemaChange(any(UUID.class), any(boolean.class), + any(SlackNotificationConfiguration.class), any(String.class)); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivityTest.java index ded19de0a6c2..3e464713edb6 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/NotifySchemaChangeActivityTest.java @@ -5,32 +5,44 @@ package io.airbyte.workers.temporal.scheduling.activities; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import io.airbyte.config.Notification; +import io.airbyte.config.SlackNotificationConfiguration; import io.airbyte.notification.SlackNotificationClient; import java.io.IOException; import java.util.UUID; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +@Slf4j class NotifySchemaChangeActivityTest { static private SlackNotificationClient mNotificationClient; static private NotifySchemaChangeActivityImpl notifySchemaChangeActivity; + static private Notification mNotification; @BeforeEach void setUp() { mNotificationClient = mock(SlackNotificationClient.class); - notifySchemaChangeActivity = new NotifySchemaChangeActivityImpl(); + mNotification = mock(Notification.class); + notifySchemaChangeActivity = spy(new NotifySchemaChangeActivityImpl()); } @Test void testNotifySchemaChange() throws IOException, InterruptedException { UUID connectionId = UUID.randomUUID(); + String connectionUrl = "connection_url"; boolean isBreaking = false; - notifySchemaChangeActivity.notifySchemaChange(mNotificationClient, connectionId, isBreaking); - verify(mNotificationClient, times(1)).notifySchemaChange(connectionId, isBreaking); + SlackNotificationConfiguration config = new SlackNotificationConfiguration(); + when(notifySchemaChangeActivity.createNotification(config)).thenReturn(mNotification); + when(notifySchemaChangeActivity.createNotificationClient(mNotification)).thenReturn(mNotificationClient); + notifySchemaChangeActivity.notifySchemaChange(connectionId, isBreaking, config, connectionUrl); + verify(mNotificationClient, times(1)).notifySchemaChange(connectionId, isBreaking, config, connectionUrl); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index e517bebd087d..870f622891be 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -71,7 +71,7 @@ void testRefreshSchema() throws ApiException { UUID connectionId = UUID.randomUUID(); refreshSchemaActivity.refreshSchema(sourceId, connectionId); SourceDiscoverSchemaRequestBody requestBody = - new SourceDiscoverSchemaRequestBody().sourceId(sourceId).disableCache(true).connectionId(connectionId); + new SourceDiscoverSchemaRequestBody().sourceId(sourceId).disableCache(true).connectionId(connectionId).notifySchemaChange(true); verify(mSourceApi, times(1)).discoverSchemaForSource(requestBody); } diff --git a/build.gradle b/build.gradle index e3e36146acc8..e4e78860bdbe 100644 --- a/build.gradle +++ b/build.gradle @@ -459,7 +459,6 @@ subprojects { subproj -> } task('generate') { - dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) } dependsOn subprojects.collect { it.getTasksByName('generateComponentManifestClassFiles', true) } dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } } @@ -499,11 +498,6 @@ subprojects { } blackFormat.dependsOn licenseTask - def generateFilesTask = project.tasks.findByName('generateProtocolClassFiles') - if (generateFilesTask != null) { - licenseTask.dependsOn generateFilesTask - } - def generateManifestFilesTask = project.tasks.findByName('generateComponentManifestClassFiles') if (generateManifestFilesTask != null) { licenseTask.dependsOn generateManifestFilesTask diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 3bf7d9ff3113..b47332518c81 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -72,5 +72,4 @@ data: WORKER_STATE_STORAGE_TYPE: {{ .Values.global.state.storage.type | quote }} SHOULD_RUN_NOTIFY_WORKFLOWS: "false" MAX_NOTIFY_WORKERS: {{ .Values.worker.maxNotifyWorkers | default "5" | quote }} - {{- end }} diff --git a/deps.toml b/deps.toml index f4de3f8d7200..dde9364fa182 100644 --- a/deps.toml +++ b/deps.toml @@ -1,4 +1,5 @@ [versions] +airbyte-protocol = "1.0.0" commons_io = "2.7" connectors-destination-testcontainers-clickhouse = "1.17.3" connectors-destination-testcontainers-elasticsearch = "1.17.3" @@ -33,6 +34,7 @@ slf4j = "1.7.36" temporal = "1.17.0" [libraries] +airbyte-protocol = { module = "io.airbyte.airbyte-protocol:protocol-models", version.ref = "airbyte-protocol" } apache-commons = { module = "org.apache.commons:commons-compress", version = "1.20" } apache-commons-lang = { module = "org.apache.commons:commons-lang3", version = "3.11" } appender-log4j2 = { module = "com.therealvan:appender-log4j2", version = "3.6.0" } @@ -64,7 +66,7 @@ fasterxml = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "faste findsecbugs-plugin = { module = "com.h3xstream.findsecbugs:findsecbugs-plugin", version = "1.12.0" } flyway-core = { module = "org.flywaydb:flyway-core", version.ref = "flyway" } glassfish = { module = "org.glassfish.jersey:jackson-bom", version.ref = "glassfish_version" } -google-cloud-storage = { module = "com.google.cloud:google-cloud-storage", version = "2.2.2" } +google-cloud-storage = { module = "com.google.cloud:google-cloud-storage", version = "2.17.2" } guava = { module = "com.google.guava:guava", version = "31.1-jre" } hikaricp = { module = "com.zaxxer:HikariCP", version.ref = "hikaricp" } jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "fasterxml_version" } diff --git a/docker-compose.yaml b/docker-compose.yaml index a8baa3c3ecbf..bc91b95e1a48 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -158,6 +158,8 @@ services: - GITHUB_STORE_BRANCH=${GITHUB_STORE_BRANCH} - MICRONAUT_ENVIRONMENTS=${WORKERS_MICRONAUT_ENVIRONMENTS} - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA} + - MAX_NOTIFY_WORKERS=5 + - SHOULD_RUN_NOTIFY_WORKFLOWS=false ports: - "8001" configs: diff --git a/docs/cloud/managing-airbyte-cloud.md b/docs/cloud/managing-airbyte-cloud.md index 5ba33bd26cc6..5daf579b664f 100644 --- a/docs/cloud/managing-airbyte-cloud.md +++ b/docs/cloud/managing-airbyte-cloud.md @@ -347,23 +347,18 @@ To get notified when your source schema changes: 4. Toggle **Schema update notifications**. -## Display Connection State +## Display the connection state -**Connection State** provides additional information about incremental syncs. It includes the most recent values for the global or stream-level cursors, which can aid in debugging or determining which data will be included in the next syncs. +Connection state provides additional information about incremental syncs. It includes the most recent values for the global or stream-level cursors, which can aid in debugging or determining which data will be included in the next syncs. -To display **Connection State**: +To display the connection state: +1. On the [Airbyte Cloud](http://cloud.airbyte.io) dashboard, click **Connections** and then click the connection you want to display. -1. On the [Airbyte Cloud](http://cloud.airbyte.io) dashboard, click **Settings**. - -2. Click **General Settings**. - -3. Toggle **Enable advanced mode** and click **Save changes**. - -4. Click **Connections** in the navigation bar and then click the connection in the list you want to display. +2. Click the **Settings** tab on the Connection page. -5. Click the **Settings** tab on the Connection page. +3. Click the **Advanced** dropdown arrow. - The **Connection State** displays. + **Connection State** displays. ## Choose the data residency for a connection You can choose the data residency for your connection in the connection settings. You can also choose data residency when creating a [new connection](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud#set-up-a-connection), or you can set the [default data residency](#choose-your-default-data-residency) for your workspace. diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index feb39b876cdc..62089cd1e2ff 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -12065,6 +12065,7 @@

    SourceDiscoverSchemaRequestB
    sourceId
    UUID format: uuid
    connectionId (optional)
    UUID format: uuid
    disable_cache (optional)
    Boolean
    +
    notifySchemaChange (optional)
    Boolean

diff --git a/octavia-cli/octavia_cli/apply/resources.py b/octavia-cli/octavia_cli/apply/resources.py index f5119b8ebefb..13b9b5776d00 100644 --- a/octavia-cli/octavia_cli/apply/resources.py +++ b/octavia-cli/octavia_cli/apply/resources.py @@ -437,7 +437,6 @@ def resource_id(self) -> Optional[str]: class SourceAndDestination(BaseResource): - @property @abc.abstractmethod def definition(