From ab81e4d1a07017e17b0744dbb36d7ff56cdab633 Mon Sep 17 00:00:00 2001 From: Michael Nguyen <67665595+michaelnguyen26@users.noreply.github.com> Date: Mon, 30 May 2022 09:40:27 -1000 Subject: [PATCH 1/6] Source Mixpanel: Added "end_date" field in specs (#12801) * Added end_date for source mixpanel * Update airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json Co-authored-by: Marcos Marx * add end_date * auto-bump connector version Co-authored-by: Cheryl0402 Co-authored-by: Marcos Marx Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 11 ++++++++++- .../connectors/source-mixpanel/Dockerfile | 2 +- .../source-mixpanel/source_mixpanel/spec.json | 7 +++++++ docs/integrations/sources/mixpanel.md | 1 + 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d373e38d831c6..77b64052aaa7b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -541,7 +541,7 @@ - name: Mixpanel sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a dockerRepository: airbyte/source-mixpanel - dockerImageTag: 0.1.15 + dockerImageTag: 0.1.16 documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel icon: mixpanel.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ccdfc86624507..00baefb4d4a55 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5224,7 +5224,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-mixpanel:0.1.15" +- dockerImage: "airbyte/source-mixpanel:0.1.16" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel" connectionSpecification: @@ -5274,6 +5274,15 @@ examples: - "2021-11-16" pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" + end_date: + title: "End Date" + type: "string" + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ + \ data after this date will not be replicated. Left empty to always sync\ + \ to most recent date" + examples: + - "2021-11-16" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" region: title: "Region" description: "The region of mixpanel domain instance either US or EU." diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index 408f88e4bef83..f9f6015871649 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.15 +LABEL io.airbyte.version=0.1.16 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json index d43c4b1c28e4c..3742e684dfddb 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json @@ -39,6 +39,13 @@ "examples": ["2021-11-16"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" }, + "end_date": { + "title": "End Date", + "type": "string", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated. Left empty to always sync to most recent date", + "examples": ["2021-11-16"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" + }, "region": { "title": "Region", "description": "The region of mixpanel domain instance either US or EU.", diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index 259ec5089a46f..3d1f94f211335 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -59,6 +59,7 @@ Select the correct region \(EU or US\) for your Mixpanel project. See detail [he | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| +| `0.1.16` | 2022-05-30 | [\#12801](https://github.com/airbytehq/airbyte/pull/12801) | Add end_date parameter | | `0.1.15` | 2022-05-04 | [\#12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy | | `0.1.14` | 2022-05-02 | [11501](https://github.com/airbytehq/airbyte/pull/11501) | Improve incremental sync method to streams | | `0.1.13` | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | From ae95d2981b91b85772264eb199324df8dec97945 Mon Sep 17 00:00:00 2001 From: Eugene Date: Mon, 30 May 2022 23:32:51 +0300 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=90=9BDestination-dynamodb:=20fixed?= =?UTF-8?q?=20build's=20run=20(#13303)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [13180] Destination-dynamodb: fixed build's run --- .../main/resources/seed/destination_definitions.yaml | 2 +- .../src/main/resources/seed/destination_specs.yaml | 10 +++++----- .../connectors/destination-dynamodb/Dockerfile | 2 +- .../dynamodb/DynamodbDestinationAcceptanceTest.java | 5 ----- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 612fac657e178..1be031ac7b00e 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -80,7 +80,7 @@ - name: DynamoDB destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 dockerRepository: airbyte/destination-dynamodb - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/dynamodb icon: dynamodb.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 67b95e0eda1c0..45ea5d089d098 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1111,7 +1111,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-dynamodb:0.1.2" +- dockerImage: "airbyte/destination-dynamodb:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/dynamodb" connectionSpecification: @@ -1119,7 +1119,7 @@ title: "DynamoDB Destination Spec" type: "object" required: - - "dynamodb_table_name" + - "dynamodb_table_name_prefix" - "dynamodb_region" - "access_key_id" - "secret_access_key" @@ -1133,10 +1133,10 @@ \ AWS DynamoDB, just leave empty)." examples: - "http://localhost:9000" - dynamodb_table_name: - title: "DynamoDB Table Name" + dynamodb_table_name_prefix: + title: "Table name prefix" type: "string" - description: "The name of the DynamoDB table." + description: "The prefix to use when naming DynamoDB tables." examples: - "airbyte_sync" dynamodb_region: diff --git a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile index e46c220fe3059..487b941746785 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile +++ b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-dynamodb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-dynamodb diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java index 364c39e2edb4a..427e18d5d21cd 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java @@ -13,10 +13,8 @@ import com.amazonaws.services.dynamodbv2.document.*; import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.io.IOs; -import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; @@ -30,7 +28,6 @@ public class DynamodbDestinationAcceptanceTest extends DestinationAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(DynamodbDestinationAcceptanceTest.class); - protected static final ObjectMapper MAPPER = MoreMappers.initMapper(); protected final String secretFilePath = "secrets/config.json"; protected JsonNode configJson; @@ -69,7 +66,6 @@ protected List getAllSyncedObjects(final String streamName, final String n final var tableName = DynamodbOutputTableHelper.getOutputTableName(this.config.getTableNamePrefix(), streamName, namespace); final var table = dynamodb.getTable(tableName); final List items = new ArrayList(); - final List resultItems = new ArrayList(); Long maxSyncTime = 0L; try { @@ -86,7 +82,6 @@ protected List getAllSyncedObjects(final String streamName, final String n LOGGER.error(e.getMessage()); } - final Long finalMaxSyncTime = maxSyncTime; items.sort(Comparator.comparingLong(o -> ((BigDecimal) o.get(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)).longValue())); return items; From 1d1476e4932039dcee7a50a96d039f48f945893c Mon Sep 17 00:00:00 2001 From: Mainara Cavalcanti Date: Mon, 30 May 2022 17:49:20 -0300 Subject: [PATCH 3/6] Jira Source: Added resolutiondate to fields on issues stream (#13202) * Added resolutiondate to fields on issues stream (jira source) * updated jira source changelog * updated dockerfile version * correct jira ci Co-authored-by: marcosmarxm --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 2 +- airbyte-integrations/connectors/source-jira/Dockerfile | 2 +- .../connectors/source-jira/acceptance-test-config.yml | 1 + .../source-jira/source_jira/schemas/application_roles.json | 7 ++++++- .../connectors/source-jira/source_jira/schemas/epics.json | 2 +- .../source-jira/source_jira/schemas/sprint_issues.json | 4 ++-- .../connectors/source-jira/source_jira/streams.py | 3 +++ docs/integrations/sources/jira.md | 1 + 9 files changed, 17 insertions(+), 7 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 77b64052aaa7b..1af6cbe575b4b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -438,7 +438,7 @@ - name: Jira sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 dockerRepository: airbyte/source-jira - dockerImageTag: 0.2.19 + dockerImageTag: 0.2.20 documentationUrl: https://docs.airbyte.io/integrations/sources/jira icon: jira.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 00baefb4d4a55..68cd9a6b80050 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3957,7 +3957,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-jira:0.2.19" +- dockerImage: "airbyte/source-jira:0.2.20" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/jira" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-jira/Dockerfile b/airbyte-integrations/connectors/source-jira/Dockerfile index 7e071ce63ce0f..904700433c8c1 100644 --- a/airbyte-integrations/connectors/source-jira/Dockerfile +++ b/airbyte-integrations/connectors/source-jira/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.19 +LABEL io.airbyte.version=0.2.20 LABEL io.airbyte.name=airbyte/source-jira diff --git a/airbyte-integrations/connectors/source-jira/acceptance-test-config.yml b/airbyte-integrations/connectors/source-jira/acceptance-test-config.yml index 29ddbe8d6647c..69156b5d08f03 100644 --- a/airbyte-integrations/connectors/source-jira/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-jira/acceptance-test-config.yml @@ -29,6 +29,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: ["epics", "screen_tab_fields", "sprint_issues", "sprints"] + timeout_seconds: 1800 # incremental: # - config_path: "secrets/config.json" # configured_catalog_path: "integration_tests/inc_configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json index 3d85229690754..e9e445eafa03f 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json @@ -50,8 +50,13 @@ "platform": { "type": "boolean", "description": "Indicates if the application role belongs to Jira platform (`jira-core`)." + }, + "groupDetails": { + "type": ["null", "array"], + "description": "Group Details", + "items": { "type": ["null", "object"] } } }, - "additionalProperties": false, + "additionalProperties": true, "description": "Details of an application role." } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/epics.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/epics.json index 6f104238e31eb..557df798dc07d 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/epics.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/epics.json @@ -34,7 +34,7 @@ "description": "Epic summary" }, "description": { - "type": ["string", "null"], + "type": ["string", "null", "object"], "description": "Epic description" }, "status": { diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprint_issues.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprint_issues.json index d57da645f6a20..74f8d73643da1 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprint_issues.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprint_issues.json @@ -57,10 +57,10 @@ } }, "issueId": { - "type": "number" + "type": "string" }, "sprintId": { - "type": "number" + "type": "integer" } } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index a9da4cb6752ab..156899a99c2a3 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -373,6 +373,7 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg "parent", "priority", "project", + "resolutiondate", "security", "status", "subtasks", @@ -530,6 +531,8 @@ class IssueProperties(StartDateJiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-properties/#api-rest-api-3-issue-issueidorkey-properties-propertykey-get """ + primary_key = "key" + def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: key = stream_slice["key"] issue_key = stream_slice["issue_key"] diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index 1b95d42107e07..12473f54df8ac 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -95,6 +95,7 @@ The Jira connector should not run into Jira API limitations under normal usage. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.20 | 2022-05-25 | [\#13202](https://github.com/airbytehq/airbyte/pull/13202) | Adds resolutiondate to `fields` object on `issues` stream | | 0.2.19 | 2022-05-04 | [\#10835](https://github.com/airbytehq/airbyte/pull/10835) | Change description for array fields | | 0.2.18 | 2021-12-23 | [\#7378](https://github.com/airbytehq/airbyte/pull/7378) | Adds experimental endpoint Pull Request | | 0.2.17 | 2021-12-23 | [\#9079](https://github.com/airbytehq/airbyte/pull/9079) | Update schema for `filters` stream + fix fetching `filters` stream | From ae2aea8b12bf05926682d231690d0f63124607db Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Tue, 31 May 2022 10:49:41 +0530 Subject: [PATCH 4/6] feat: connection name can be edited through UI (#12803) * feat: connection name can be edited through UI * fix: changes requested by edmundito * 1. feat: Added connection name edit in connection settings 2. feat: Added connection name in listing connections page 3. fix: requested changes * chore: handle UI and e2e tests * chore: fix e2e tests for webapp * fix: UI margins * fix: webe2e tests * fix: input is not focused when clicked on name * fix: disable input while making API call * feat: added enter/esc functionality to connection name edit --- .../cypress/integration/connection.spec.js | 3 - .../cypress/support/commands/connection.js | 1 + .../ConnectorCard/ConnectorCard.tsx | 72 +++++++ .../src/components/ConnectorCard/index.tsx | 3 + .../EntityTable/ConnectionTable.tsx | 32 +++- .../src/components/EntityTable/types.ts | 1 + .../src/components/EntityTable/utils.tsx | 1 + .../src/components/base/Input/Input.tsx | 13 +- airbyte-webapp/src/components/index.tsx | 1 + .../src/hooks/services/useConnectionHook.tsx | 1 + airbyte-webapp/src/locales/en.json | 3 +- .../ConnectionItemPage/ConnectionItemPage.tsx | 6 +- .../components/ConnectionName.tsx | 177 ++++++++++++++++++ .../components/ConnectionPageTitle.tsx | 44 ++--- .../components/StatusMainInfo.tsx | 140 +++++++------- .../components/StatusView.tsx | 21 +-- .../src/utils/addEnterEscFuncForInput.tsx | 28 +++ .../ConnectionForm/ConnectionForm.tsx | 38 +++- .../Connection/ConnectionForm/formConfig.tsx | 3 + 19 files changed, 458 insertions(+), 130 deletions(-) create mode 100644 airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx create mode 100644 airbyte-webapp/src/components/ConnectorCard/index.tsx create mode 100644 airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx create mode 100644 airbyte-webapp/src/utils/addEnterEscFuncForInput.tsx diff --git a/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.js b/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.js index 29cabfe394854..a20c0f9b7710a 100644 --- a/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.js +++ b/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.js @@ -26,9 +26,6 @@ describe("Connection main actions", () => { cy.get("button[type=submit]").first().click(); cy.wait("@updateConnection"); cy.get("span[data-id='success-result']").should("exist"); - - cy.get("div[data-id='status-step']").click(); - cy.get("div").contains("5 min").should("exist"); }); it("Delete connection", () => { diff --git a/airbyte-webapp-e2e-tests/cypress/support/commands/connection.js b/airbyte-webapp-e2e-tests/cypress/support/commands/connection.js index ced7ea833bd05..a207f53ca17f4 100644 --- a/airbyte-webapp-e2e-tests/cypress/support/commands/connection.js +++ b/airbyte-webapp-e2e-tests/cypress/support/commands/connection.js @@ -11,6 +11,7 @@ Cypress.Commands.add("createTestConnection", (sourceName, destinationName) => { cy.wait("@discoverSchema"); + cy.get("div[data-testid='connectionName']").type("Connection name"); cy.get("div[data-testid='schedule']").click(); cy.get("div[data-testid='manual']").click(); diff --git a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx new file mode 100644 index 0000000000000..56685c8f78589 --- /dev/null +++ b/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx @@ -0,0 +1,72 @@ +import styled from "styled-components"; + +import { ReleaseStageBadge } from "components/ReleaseStageBadge"; + +import { ReleaseStage } from "core/request/AirbyteClient"; +import { getIcon } from "utils/imageUtils"; + +type Props = { + connectionName: string; + icon?: string; + connectorName: string; + releaseStage?: ReleaseStage; +}; + +const MainComponent = styled.div` + display: flex; + padding: 10px; + width: 220px; + align-items: center; +`; + +const Details = styled.div` + width: 160px; + margin-left: 10px; + display: flex; + flex-direction: column; + font-weight: normal; +`; + +const EntityIcon = styled.div` + height: 30px; + width: 30px; +`; + +const ConnectionName = styled.div` + font-size: 14px; + color: #1a194d; + text-align: left; + margin-right: 10px; +`; + +const ConnectorDetails = styled.div` + display: flex; + justify-content: flex-start; + align-items: center; +`; + +const ConnectorName = styled.div` + font-size: 11px; + margin-top: 1px; + color: #afafc1; + text-align: left; +`; + +function ConnectorCard(props: Props) { + const { connectionName, connectorName, icon, releaseStage } = props; + + return ( + + {icon && {getIcon(icon)}} +
+ + {connectionName} + {releaseStage && } + + {connectorName} +
+ + ); +} + +export default ConnectorCard; diff --git a/airbyte-webapp/src/components/ConnectorCard/index.tsx b/airbyte-webapp/src/components/ConnectorCard/index.tsx new file mode 100644 index 0000000000000..18dd1ffc7d665 --- /dev/null +++ b/airbyte-webapp/src/components/ConnectorCard/index.tsx @@ -0,0 +1,3 @@ +import ConnectorCard from "./ConnectorCard"; + +export default ConnectorCard; diff --git a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx index fabca8270de50..cd9ff5ddf17fc 100644 --- a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx +++ b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx @@ -35,7 +35,7 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onChangeS const { hasFeature } = useFeatureService(); const allowSync = hasFeature(FeatureItem.AllowSync); - const sortBy = query.sortBy || "entity"; + const sortBy = query.sortBy || "entityName"; const sortOrder = query.order || SortOrderEnum.ASC; const onSortClick = useCallback( @@ -61,7 +61,7 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onChangeS if (sortBy === "lastSync") { result = b[sortBy] - a[sortBy]; } else { - result = a[`${sortBy}Name`].toLowerCase().localeCompare(b[`${sortBy}Name`].toLowerCase()); + result = a[sortBy].toLowerCase().localeCompare(b[sortBy].toLowerCase()); } if (sortOrder === SortOrderEnum.DESC) { @@ -77,6 +77,24 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onChangeS const columns = React.useMemo( () => [ + { + Header: ( + <> + + onSortClick("name")} + /> + + ), + headerHighlighted: true, + accessor: "name", + customWidth: 30, + Cell: ({ cell, row }: CellProps) => ( + + ), + }, { Header: ( <> @@ -86,20 +104,18 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onChangeS )} onSortClick("entity")} + onClick={() => onSortClick("entityName")} /> ), headerHighlighted: true, accessor: "entityName", - customWidth: 40, Cell: ({ cell, row }: CellProps) => ( @@ -114,9 +130,9 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onChangeS )} onSortClick("connector")} + onClick={() => onSortClick("connectorName")} /> ), diff --git a/airbyte-webapp/src/components/EntityTable/types.ts b/airbyte-webapp/src/components/EntityTable/types.ts index 2c0eaeebdf374..a32faa143d4b7 100644 --- a/airbyte-webapp/src/components/EntityTable/types.ts +++ b/airbyte-webapp/src/components/EntityTable/types.ts @@ -17,6 +17,7 @@ type EntityTableDataItem = { type ITableDataItem = { connectionId: string; + name: string; entityName: string; connectorName: string; enabled: boolean; diff --git a/airbyte-webapp/src/components/EntityTable/utils.tsx b/airbyte-webapp/src/components/EntityTable/utils.tsx index 990bd9aee60a2..dab499bc0136f 100644 --- a/airbyte-webapp/src/components/EntityTable/utils.tsx +++ b/airbyte-webapp/src/components/EntityTable/utils.tsx @@ -90,6 +90,7 @@ export const getConnectionTableData = ( return { connectionId: connection.connectionId, + name: connection.name, entityName: type === "connection" ? `${connection.source?.sourceName} - ${connection.source?.name}` diff --git a/airbyte-webapp/src/components/base/Input/Input.tsx b/airbyte-webapp/src/components/base/Input/Input.tsx index 117ff76e206a5..5fea6391dbd55 100644 --- a/airbyte-webapp/src/components/base/Input/Input.tsx +++ b/airbyte-webapp/src/components/base/Input/Input.tsx @@ -1,6 +1,6 @@ import { faEye, faEyeSlash } from "@fortawesome/free-regular-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import React from "react"; +import React, { useEffect, useRef } from "react"; import { useIntl } from "react-intl"; import { useToggle } from "react-use"; import styled from "styled-components"; @@ -23,6 +23,7 @@ const getBackgroundColor = (props: IStyleProps) => { export interface InputProps extends React.InputHTMLAttributes { error?: boolean; light?: boolean; + defaultFocus?: boolean; } const InputContainer = styled.div` @@ -78,7 +79,10 @@ const VisibilityButton = styled(Button)` `; const Input: React.FC = (props) => { + const { defaultFocus = false } = props; + const { formatMessage } = useIntl(); + const inputRef = useRef(null); const [isContentVisible, setIsContentVisible] = useToggle(false); const [focused, toggleFocused] = useToggle(false); @@ -87,10 +91,17 @@ const Input: React.FC = (props) => { const type = isPassword ? (isContentVisible ? "text" : "password") : props.type; const onInputFocusChange = () => toggleFocused(); + useEffect(() => { + if (defaultFocus && inputRef.current !== null) { + inputRef.current.focus(); + } + }, [inputRef, defaultFocus]); + return ( { const connectionId = params.connectionId || ""; const currentStep = params["*"] || ConnectionSettingsRoutes.STATUS; const connection = useGetConnection(connectionId); + const [isStatusUpdating, setStatusUpdating] = useState(false); const { source, destination } = connection; @@ -68,6 +69,7 @@ const ConnectionItemPage: React.FC = () => { destination={destination} connection={connection} currentStep={currentStep} + onStatusUpdating={setStatusUpdating} /> } error={ @@ -80,7 +82,7 @@ const ConnectionItemPage: React.FC = () => { } + element={} /> theme.primaryColor}; +`; + +const NameContainer = styled.div` + width: 650px; + background-color: rgba(255, 235, 215, 0.4); + display: flex; + align-items: center; + position: relative; + padding: 0 20px; + border-radius: 8px; + border: 1px solid rgba(255, 235, 215, 0.4); + + &:hover { + cursor: pointer; + border: ${({ theme }) => `1px solid ${theme.primaryColor}`}; + background-color: ${({ theme }) => theme.primaryColor12}; + } + + &:hover ${Icon} { + display: block; + } +`; + +const EditingContainer = styled.div` + width: 650px; + display: flex; + background-color: white; + justify-content: center; + align-items: center; + border-radius: 8px; + border: ${({ theme }) => `1px solid ${theme.primaryColor}`}; +`; + +const InputContainer = styled.div` + height: 50px; + width: 100%; + + div { + border-radius: 8px; + border: none; + box-shadow: none; + background-color: white !important; + } +`; + +const Name = styled.div` + flex-grow: 1; +`; + +const H2 = styled.h2` + font-weight: 700; + font-size: 24px; + line-height: 29px; + text-align: center; + color: #1a194d; + margin: 10px; +`; + +const StyledInput = styled(Input)` + border-radius: 8px; + background-color: white; + font-size: 24px; + height: 50px; + div { + border: none; + } +`; + +const InputWithKeystroke = addEnterEscFuncForInput(StyledInput); + +const ConnectionName: React.FC = ({ connection }) => { + const { name } = connection; + const [editingState, setEditingState] = useState(false); + const [loading, setLoading] = useState(false); + const [connectionName, setConnectionName] = useState(connection.name); + const { mutateAsync: updateConnection } = useUpdateConnection(); + + const setEditing = () => { + setEditingState(true); + }; + + const inputChange = (event: ChangeEvent) => { + const value = event.currentTarget.value; + if (value) { + setConnectionName(event.currentTarget.value); + } + }; + + const onEscape = () => { + setEditingState(false); + setConnectionName(name); + }; + + const onEnter = async () => { + await updateConnectionAsync(); + }; + + const onBlur = async () => { + await updateConnectionAsync(); + }; + + const updateConnectionAsync = async () => { + // Update only when the name is changed + if (connection.name !== connectionName) { + setLoading(true); + await updateConnection({ + name: connectionName, + connectionId: connection.connectionId, + namespaceDefinition: connection.namespaceDefinition, + syncCatalog: connection.syncCatalog, + status: connection.status, + prefix: connection.prefix, + }); + setLoading(false); + } + + setEditingState(false); + }; + + return ( + + {!editingState && ( + + +

{name}

+ + + + )} + {editingState && ( + + + + + + )} + + ); +}; + +export default ConnectionName; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx index c0273474b6331..8fcdb8cd8f9c5 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx @@ -1,23 +1,23 @@ -import { faArrowRight } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { H6, Link } from "components"; +import { H6 } from "components"; import StepsMenu from "components/StepsMenu"; import { ConnectionStatus, DestinationRead, SourceRead, WebBackendConnectionRead } from "core/request/AirbyteClient"; import useRouter from "hooks/useRouter"; -import { RoutePaths } from "../../../../routePaths"; import { ConnectionSettingsRoutes } from "../ConnectionSettingsRoutes"; +import ConnectionName from "./ConnectionName"; +import { StatusMainInfo } from "./StatusMainInfo"; interface ConnectionPageTitleProps { source: SourceRead; destination: DestinationRead; connection: WebBackendConnectionRead; currentStep: ConnectionSettingsRoutes; + onStatusUpdating?: (updating: boolean) => void; } const Title = styled.div` @@ -26,23 +26,21 @@ const Title = styled.div` `; const Links = styled.div` - margin-bottom: 18px; + margin: 18px 0; font-size: 15px; font-weight: bold; + display: flex; + align-items: center; + justify-content: center; `; -const ConnectorsLink = styled(Link)` - font-style: normal; - font-weight: bold; - font-size: 24px; - line-height: 29px; - text-align: center; - display: inline-block; - margin: 0 16px; - color: ${({ theme }) => theme.textColor}; -`; - -const ConnectionPageTitle: React.FC = ({ source, destination, connection, currentStep }) => { +const ConnectionPageTitle: React.FC = ({ + source, + destination, + connection, + currentStep, + onStatusUpdating, +}) => { const { push } = useRouter<{ id: string }>(); const steps = [ @@ -79,12 +77,14 @@ const ConnectionPageTitle: React.FC = ({ source, desti
+ - {source.name} - - - {destination.name} - + diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.tsx index e88f0a957efa8..5d19a865060b3 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.tsx @@ -1,96 +1,92 @@ +import { faArrowRight } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; -import { FormattedMessage } from "react-intl"; +import { Link as ReactLink } from "react-router-dom"; import styled from "styled-components"; -import { ContentCard } from "components"; -import { ConnectorIcon } from "components/ConnectorIcon"; -import { ReleaseStageBadge } from "components/ReleaseStageBadge"; -import { Cell, Header, Row } from "components/SimpleTableComponents"; +import ConnectorCard from "components/ConnectorCard"; -import { - ConnectionStatus, - DestinationDefinitionRead, - SourceDefinitionRead, - WebBackendConnectionRead, -} from "core/request/AirbyteClient"; +import FrequencyConfig from "config/FrequencyConfig.json"; +import { ConnectionStatus, SourceRead, DestinationRead, WebBackendConnectionRead } from "core/request/AirbyteClient"; +import { FeatureItem, useFeatureService } from "hooks/services/Feature"; +import { RoutePaths } from "pages/routePaths"; +import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; +import { useSourceDefinition } from "services/connector/SourceDefinitionService"; +import { equal } from "utils/objects"; import EnabledControl from "./EnabledControl"; -const MainInfo = styled(ContentCard)` - margin-bottom: 14px; - padding: 23px 20px 20px 23px; -`; - -const Img = styled(ConnectorIcon)` - display: inline-block; -`; - -const SourceCell = styled(Cell)` +const MainContainer = styled.div` + width: 650px; display: flex; + justify-content: space-between; + padding: 10px 20px; + background-color: white; + border-radius: 10px; align-items: center; - gap: 6px; `; -const EnabledCell = styled(Cell)` - display: flex; - align-items: center; - margin-top: -18px; +const ConnectorsLink = styled(ReactLink)` + cursor: pointer; + text-decoration: none; + border-radius: 10px; + + &:hover { + background-color: ${({ theme }) => theme.greyColor10}; + } `; interface StatusMainInfoProps { connection: WebBackendConnectionRead; - frequencyText?: string; - destinationDefinition?: DestinationDefinitionRead; - sourceDefinition?: SourceDefinitionRead; - allowSync?: boolean; + source: SourceRead; + destination: DestinationRead; onStatusUpdating?: (updating: boolean) => void; } export const StatusMainInfo: React.FC = ({ - connection, - frequencyText, - destinationDefinition, - sourceDefinition, - allowSync, onStatusUpdating, + connection, + source, + destination, }) => { + const { hasFeature } = useFeatureService(); + + const sourceDefinition = useSourceDefinition(source.sourceDefinitionId); + const destinationDefinition = useDestinationDefinition(destination.destinationDefinitionId); + + const allowSync = hasFeature(FeatureItem.AllowSync); + const frequency = FrequencyConfig.find((item) => equal(item.config, connection.schedule)); + + const sourceConnectionPath = `../../${RoutePaths.Source}/${source.sourceId}`; + const destinationConnectionPath = `../../${RoutePaths.Destination}/${destination.destinationId}`; + return ( - -
- - - - - - - - - - {connection.status !== ConnectionStatus.deprecated && } -
- - - - {connection.source?.sourceName} - - - - - {connection.destination?.destinationName} - - - {frequencyText} - {connection.status !== ConnectionStatus.deprecated && ( - - - - )} - - + + + + + + + + + {connection.status !== ConnectionStatus.deprecated && ( + + )} + ); }; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx index 178a85b153fbb..e419c14cedf74 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx @@ -11,17 +11,14 @@ import ResetDataModal from "components/ResetDataModal"; import { FeatureItem, useFeatureService } from "hooks/services/Feature"; import { useResetConnection, useSyncConnection } from "hooks/services/useConnectionHook"; import useLoadingState from "hooks/useLoadingState"; -import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; -import { useSourceDefinition } from "services/connector/SourceDefinitionService"; import { useListJobs } from "services/job/JobService"; import { ConnectionStatus, WebBackendConnectionRead } from "../../../../../core/request/AirbyteClient"; import JobsList from "./JobsList"; -import { StatusMainInfo } from "./StatusMainInfo"; interface StatusViewProps { connection: WebBackendConnectionRead; - frequencyText?: string; + isStatusUpdating?: boolean; } const Content = styled.div` @@ -51,23 +48,17 @@ const SyncButton = styled(LoadingButton)` min-height: 28px; `; -const StatusView: React.FC = ({ connection, frequencyText }) => { +const StatusView: React.FC = ({ connection, isStatusUpdating }) => { const [isModalOpen, setIsModalOpen] = useState(false); const { isLoading, showFeedback, startAction } = useLoadingState(); const { hasFeature } = useFeatureService(); const allowSync = hasFeature(FeatureItem.AllowSync); - const sourceDefinition = useSourceDefinition(connection?.source.sourceDefinitionId); - - const destinationDefinition = useDestinationDefinition(connection.destination.destinationDefinitionId); - const jobs = useListJobs({ configId: connection.connectionId, configTypes: ["sync", "reset_connection"], }); - const [isStatusUpdating, setStatusUpdating] = useState(false); - const { mutateAsync: resetConnection } = useResetConnection(); const { mutateAsync: syncConnection } = useSyncConnection(); @@ -76,14 +67,6 @@ const StatusView: React.FC = ({ connection, frequencyText }) => return ( - diff --git a/airbyte-webapp/src/utils/addEnterEscFuncForInput.tsx b/airbyte-webapp/src/utils/addEnterEscFuncForInput.tsx new file mode 100644 index 0000000000000..c10eee45bf3c1 --- /dev/null +++ b/airbyte-webapp/src/utils/addEnterEscFuncForInput.tsx @@ -0,0 +1,28 @@ +import * as React from "react"; + +export default function addEnterEscFuncForInput(WrapperComponent: React.FC) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (props: any) => { + const onKeyDown = (event: React.KeyboardEvent) => { + // Escape Key Event + if (event.key === "Escape") { + if (props.onEscape) { + props.onEscape(event); + } + } + + // Enter Key Event + if (event.key === "Enter") { + if (props.onEnter) { + props.onEnter(event); + } + } + + if (props.onKeyDown) { + props.onKeyDown(event); + } + }; + + return ; + }; +} diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx index 0ffd7661fd8fb..6a10b436b57f0 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx @@ -138,9 +138,10 @@ const ConnectionForm: React.FC = ({ const formatMessage = useIntl().formatMessage; - const initialValues = useInitialValues(connection, destDefinition, mode !== "create"); - + const isEditMode: boolean = mode !== "create"; + const initialValues = useInitialValues(connection, destDefinition, isEditMode); const workspace = useCurrentWorkspace(); + const onFormSubmit = useCallback( async (values: FormikConnectionFormValues, formikHelpers: FormikHelpers) => { const formValues: ConnectionFormValues = connectionValidationSchema.cast(values, { @@ -192,6 +193,39 @@ const ConnectionForm: React.FC = ({ {({ isSubmitting, setFieldValue, isValid, dirty, resetForm, values }) => ( + {!isEditMode && ( + + + {({ field, meta }: FieldProps) => ( + + + + + + + + + )} + + + )}
}> {({ field, meta }: FieldProps) => ( diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx index 62bcbf2fe536d..71c11f5f2f68e 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx @@ -30,6 +30,7 @@ import { } from "../../../core/request/AirbyteClient"; type FormikConnectionFormValues = { + name?: string; schedule?: ConnectionSchedule | null; prefix: string; syncCatalog: SyncSchema; @@ -71,6 +72,7 @@ function useDefaultTransformation(): OperationCreate { const connectionValidationSchema = yup .object({ + name: yup.string().required("form.empty.error"), schedule: yup .object({ units: yup.number().required("form.empty.error"), @@ -277,6 +279,7 @@ const useInitialValues = ( return useMemo(() => { const initialValues: FormikConnectionFormValues = { + name: connection.name ?? `${connection.source.name} <> ${connection.destination.name}`, syncCatalog: initialSchema, schedule: connection.schedule !== undefined ? connection.schedule : DEFAULT_SCHEDULE, prefix: connection.prefix || "", From fc73df424b5374fa8c2e51548f3c81e004c287fb Mon Sep 17 00:00:00 2001 From: YI CHEN <35242621+Cheryl0402@users.noreply.github.com> Date: Mon, 30 May 2022 23:22:11 -0700 Subject: [PATCH 5/6] Added a new unit test for getDeploymentMode() in the EnvConfigs.java (#13010) --- .../java/io/airbyte/config/EnvConfigsTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java index 734682329ea8a..995e74f6f6673 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java @@ -177,6 +177,21 @@ void testTrackingStrategy() { assertEquals(Configs.TrackingStrategy.LOGGING, config.getTrackingStrategy()); } + @Test + void testDeploymentMode() { + envMap.put(EnvConfigs.DEPLOYMENT_MODE, null); + assertEquals(Configs.DeploymentMode.OSS, config.getDeploymentMode()); + + envMap.put(EnvConfigs.DEPLOYMENT_MODE, "CLOUD"); + assertEquals(Configs.DeploymentMode.CLOUD, config.getDeploymentMode()); + + envMap.put(EnvConfigs.DEPLOYMENT_MODE, "oss"); + assertEquals(Configs.DeploymentMode.OSS, config.getDeploymentMode()); + + envMap.put(EnvConfigs.DEPLOYMENT_MODE, "OSS"); + assertEquals(Configs.DeploymentMode.OSS, config.getDeploymentMode()); + } + @Test void testworkerKubeTolerations() { final String airbyteServer = "airbyte-server"; From c55f185109d2c275271e6c13655ca37fa08a5aed Mon Sep 17 00:00:00 2001 From: Ahmed Buksh <38909200+ahmed-buksh@users.noreply.github.com> Date: Tue, 31 May 2022 11:42:46 +0500 Subject: [PATCH 6/6] :rocket: Source Klaviyo: New Stream addition along with update to existing (#11685) * :rocket: flow stream added along with flow, campaign and flow message addition to event * :zap: liniting fix * :hammer: annotations updated along with update method for klaviyo * :boom: docker version updated and log added * :hammer: fixed acceptance test for klaviyo flow stream * :hammer: unused import removed * fix: flows stream has no records thus tests are failing * chore: update seed file Co-authored-by: Harshith Mullapudi --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-klaviyo/Dockerfile | 2 +- .../source-klaviyo/acceptance-test-config.yml | 1 + .../integration_tests/abnormal_state.json | 3 ++ .../integration_tests/configured_catalog.json | 15 ++++++++ .../source-klaviyo/source_klaviyo/schemas.py | 13 +++++++ .../source-klaviyo/source_klaviyo/source.py | 3 +- .../source-klaviyo/source_klaviyo/streams.py | 34 +++++++++++++++++-- docs/integrations/sources/klaviyo.md | 1 + 10 files changed, 70 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1af6cbe575b4b..3e02e635e9ef7 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -454,7 +454,7 @@ - name: Klaviyo sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde dockerRepository: airbyte/source-klaviyo - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/klaviyo icon: klaviyo.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 68cd9a6b80050..0ee0b7d60cd9a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4282,7 +4282,7 @@ supported_destination_sync_modes: [] supported_source_sync_modes: - "append" -- dockerImage: "airbyte/source-klaviyo:0.1.3" +- dockerImage: "airbyte/source-klaviyo:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/klaviyo" changelogUrl: "https://docs.airbyte.io/integrations/sources/klaviyo" diff --git a/airbyte-integrations/connectors/source-klaviyo/Dockerfile b/airbyte-integrations/connectors/source-klaviyo/Dockerfile index 316f3e986371b..82a5c6bc8d422 100644 --- a/airbyte-integrations/connectors/source-klaviyo/Dockerfile +++ b/airbyte-integrations/connectors/source-klaviyo/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-klaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml index a2bd973a4e927..8294d0bf070c4 100644 --- a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml @@ -13,6 +13,7 @@ tests: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" + empty_streams: ['flows'] incremental: - config_path: "secrets/config.json" future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json index 8a88bd61549a9..c9ac3cf70e411 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json @@ -4,5 +4,8 @@ }, "global_exclusions": { "timestamp": "2120-10-10T00:00:00Z" + }, + "flows": { + "created": "2120-10-10 00:00:00" } } diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json index ac14b2cb00cc5..3e5f3a2f9a0a0 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json @@ -59,6 +59,21 @@ "cursor_field": null, "destination_sync_mode": "append", "primary_key": null + }, + { + "stream": { + "name": "flows", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null } ] } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas.py index 3a610b9fe6c4f..e36c6a2c456d3 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas.py @@ -104,6 +104,19 @@ class Event(BaseSchemaModel): statistic_id: str event_properties: dict person: dict + flow_id: Optional[str] + campaign_id: Optional[str] + flow_message_id: Optional[str] + + +class Flow(BaseSchemaModel): + id: str + name: str + status: str + created: datetime + updated: datetime + customer_filter: Optional[dict] + trigger: dict class GlobalExclusion(BaseSchemaModel): diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py index 4c9d2955b620e..3edd8f9a3a904 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py @@ -10,7 +10,7 @@ from airbyte_cdk.sources.streams import Stream from pydantic import Field from pydantic.main import BaseModel -from source_klaviyo.streams import Campaigns, Events, GlobalExclusions, Lists, Metrics +from source_klaviyo.streams import Campaigns, Events, Flows, GlobalExclusions, Lists, Metrics class ConnectorConfig(BaseModel): @@ -61,6 +61,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: GlobalExclusions(api_key=config.api_key, start_date=config.start_date), Lists(api_key=config.api_key), Metrics(api_key=config.api_key), + Flows(api_key=config.api_key, start_date=config.start_date), ] def spec(self, *args, **kwargs) -> ConnectorSpecification: diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index 4b6c873ac7718..67e2e14582eb4 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -2,13 +2,14 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import datetime from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import pendulum import requests from airbyte_cdk.sources.streams.http import HttpStream -from source_klaviyo.schemas import Campaign, Event, GlobalExclusion, Metric, PersonList +from source_klaviyo.schemas import Campaign, Event, Flow, GlobalExclusion, Metric, PersonList class KlaviyoStream(HttpStream, ABC): @@ -99,7 +100,13 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. """ state_ts = int(current_stream_state.get(self.cursor_field, 0)) - return {self.cursor_field: max(latest_record.get(self.cursor_field), state_ts)} + latest_record = latest_record.get(self.cursor_field) + + if isinstance(latest_record, str): + latest_record = datetime.datetime.strptime(latest_record, "%Y-%m-%d %H:%M:%S") + latest_record = datetime.datetime.timestamp(latest_record) + + return {self.cursor_field: max(latest_record, state_ts)} def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ @@ -240,3 +247,26 @@ class Events(IncrementalKlaviyoStream): def path(self, **kwargs) -> str: return "metrics/timeline" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + :return an iterable containing each record in the response + """ + response_json = response.json() + for record in response_json.get("data", []): + flow = record["event_properties"].get("$flow") + flow_message_id = record["event_properties"].get("$message") + + record["flow_id"] = flow + record["flow_message_id"] = flow_message_id + record["campaign_id"] = flow_message_id if not flow else None + + yield record + + +class Flows(ReverseIncrementalKlaviyoStream): + schema = Flow + cursor_field = "created" + + def path(self, **kwargs) -> str: + return "flows" diff --git a/docs/integrations/sources/klaviyo.md b/docs/integrations/sources/klaviyo.md index 0c19b99bce611..c63b02da3b25d 100644 --- a/docs/integrations/sources/klaviyo.md +++ b/docs/integrations/sources/klaviyo.md @@ -52,5 +52,6 @@ Please follow these [steps](https://help.klaviyo.com/hc/en-us/articles/115005062 | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| `0.1.4` | 2022-04-15 | [11723](https://github.com/airbytehq/airbyte/issues/11723) | Enhance klaviyo source for flows stream and update to events stream. | | `0.1.3` | 2021-12-09 | [8592](https://github.com/airbytehq/airbyte/pull/8592) | Improve performance, make Global Exclusions stream incremental and enable Metrics stream. | | `0.1.2` | 2021-10-19 | [6952](https://github.com/airbytehq/airbyte/pull/6952) | Update schema validation in SAT |