Merge branch 'master' into ddavydov/#12591-source-zendesk-support-upd…

…-cdk-version

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -80,7 +80,7 @@
 - name: DynamoDB
   destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89
   dockerRepository: airbyte/destination-dynamodb
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.io/integrations/destinations/dynamodb
   icon: dynamodb.svg
   releaseStage: alpha

airbyte-config/init/src/main/resources/seed/destination_specs.yaml

@@ -1111,15 +1111,15 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-dynamodb:0.1.2"
+- dockerImage: "airbyte/destination-dynamodb:0.1.3"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/dynamodb"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "DynamoDB Destination Spec"
       type: "object"
       required:
-      - "dynamodb_table_name"
+      - "dynamodb_table_name_prefix"
       - "dynamodb_region"
       - "access_key_id"
       - "secret_access_key"
@@ -1133,10 +1133,10 @@
             \ AWS DynamoDB, just leave empty)."
           examples:
           - "http://localhost:9000"
-        dynamodb_table_name:
-          title: "DynamoDB Table Name"
+        dynamodb_table_name_prefix:
+          title: "Table name prefix"
           type: "string"
-          description: "The name of the DynamoDB table."
+          description: "The prefix to use when naming DynamoDB tables."
           examples:
           - "airbyte_sync"
         dynamodb_region:

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -438,7 +438,7 @@
 - name: Jira
   sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993
   dockerRepository: airbyte/source-jira
-  dockerImageTag: 0.2.19
+  dockerImageTag: 0.2.20
   documentationUrl: https://docs.airbyte.io/integrations/sources/jira
   icon: jira.svg
   sourceType: api
@@ -454,7 +454,7 @@
 - name: Klaviyo
   sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde
   dockerRepository: airbyte/source-klaviyo
-  dockerImageTag: 0.1.3
+  dockerImageTag: 0.1.4
   documentationUrl: https://docs.airbyte.io/integrations/sources/klaviyo
   icon: klaviyo.svg
   sourceType: api
@@ -541,7 +541,7 @@
 - name: Mixpanel
   sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a
   dockerRepository: airbyte/source-mixpanel
-  dockerImageTag: 0.1.15
+  dockerImageTag: 0.1.16
   documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel
   icon: mixpanel.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -3957,7 +3957,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-jira:0.2.19"
+- dockerImage: "airbyte/source-jira:0.2.20"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/jira"
     connectionSpecification:
@@ -4282,7 +4282,7 @@
     supported_destination_sync_modes: []
     supported_source_sync_modes:
     - "append"
-- dockerImage: "airbyte/source-klaviyo:0.1.3"
+- dockerImage: "airbyte/source-klaviyo:0.1.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/klaviyo"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/klaviyo"
@@ -5224,7 +5224,7 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-mixpanel:0.1.15"
+- dockerImage: "airbyte/source-mixpanel:0.1.16"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel"
     connectionSpecification:
@@ -5274,6 +5274,15 @@
           examples:
           - "2021-11-16"
           pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$"
+        end_date:
+          title: "End Date"
+          type: "string"
+          description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
+            \ data after this date will not be replicated. Left empty to always sync\
+            \ to most recent date"
+          examples:
+          - "2021-11-16"
+          pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$"
         region:
           title: "Region"
           description: "The region of mixpanel domain instance either US or EU."

airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java

@@ -177,6 +177,21 @@ void testTrackingStrategy() {
     assertEquals(Configs.TrackingStrategy.LOGGING, config.getTrackingStrategy());
   }
 
+  @Test
+  void testDeploymentMode() {
+    envMap.put(EnvConfigs.DEPLOYMENT_MODE, null);
+    assertEquals(Configs.DeploymentMode.OSS, config.getDeploymentMode());
+
+    envMap.put(EnvConfigs.DEPLOYMENT_MODE, "CLOUD");
+    assertEquals(Configs.DeploymentMode.CLOUD, config.getDeploymentMode());
+
+    envMap.put(EnvConfigs.DEPLOYMENT_MODE, "oss");
+    assertEquals(Configs.DeploymentMode.OSS, config.getDeploymentMode());
+
+    envMap.put(EnvConfigs.DEPLOYMENT_MODE, "OSS");
+    assertEquals(Configs.DeploymentMode.OSS, config.getDeploymentMode());
+  }
+
   @Test
   void testworkerKubeTolerations() {
     final String airbyteServer = "airbyte-server";

airbyte-integrations/connectors/destination-dynamodb/Dockerfile

@@ -16,5 +16,5 @@ ENV APPLICATION destination-dynamodb
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/destination-dynamodb

airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java

@@ -13,10 +13,8 @@
 import com.amazonaws.services.dynamodbv2.document.*;
 import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.airbyte.commons.io.IOs;
-import io.airbyte.commons.jackson.MoreMappers;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.integrations.base.JavaBaseConstants;
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
@@ -30,7 +28,6 @@
 public class DynamodbDestinationAcceptanceTest extends DestinationAcceptanceTest {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(DynamodbDestinationAcceptanceTest.class);
-  protected static final ObjectMapper MAPPER = MoreMappers.initMapper();
 
   protected final String secretFilePath = "secrets/config.json";
   protected JsonNode configJson;
@@ -69,7 +66,6 @@ protected List<Item> getAllSyncedObjects(final String streamName, final String n
     final var tableName = DynamodbOutputTableHelper.getOutputTableName(this.config.getTableNamePrefix(), streamName, namespace);
     final var table = dynamodb.getTable(tableName);
     final List<Item> items = new ArrayList<Item>();
-    final List<Item> resultItems = new ArrayList<Item>();
     Long maxSyncTime = 0L;
 
     try {
@@ -86,7 +82,6 @@ protected List<Item> getAllSyncedObjects(final String streamName, final String n
       LOGGER.error(e.getMessage());
     }
 
-    final Long finalMaxSyncTime = maxSyncTime;
     items.sort(Comparator.comparingLong(o -> ((BigDecimal) o.get(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)).longValue()));
 
     return items;

airbyte-integrations/connectors/source-jira/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.19
+LABEL io.airbyte.version=0.2.20
 LABEL io.airbyte.name=airbyte/source-jira

airbyte-integrations/connectors/source-jira/acceptance-test-config.yml

@@ -29,6 +29,7 @@ tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
       empty_streams: ["epics", "screen_tab_fields", "sprint_issues", "sprints"]
+      timeout_seconds: 1800
 #  incremental:
 #    - config_path: "secrets/config.json"
 #      configured_catalog_path: "integration_tests/inc_configured_catalog.json"

airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json

@@ -50,8 +50,13 @@
     "platform": {
       "type": "boolean",
       "description": "Indicates if the application role belongs to Jira platform (`jira-core`)."
+    },
+    "groupDetails": {
+      "type": ["null", "array"],
+      "description": "Group Details",
+      "items": { "type": ["null", "object"] }
     }
   },
-  "additionalProperties": false,
+  "additionalProperties": true,
   "description": "Details of an application role."
 }

airbyte-integrations/connectors/source-jira/source_jira/schemas/epics.json

@@ -34,7 +34,7 @@
           "description": "Epic summary"
         },
         "description": {
-          "type": ["string", "null"],
+          "type": ["string", "null", "object"],
           "description": "Epic description"
         },
         "status": {

airbyte-integrations/connectors/source-jira/source_jira/schemas/sprint_issues.json

@@ -57,10 +57,10 @@
       }
     },
     "issueId": {
-      "type": "number"
+      "type": "string"
     },
     "sprintId": {
-      "type": "number"
+      "type": "integer"
     }
   }
 }

airbyte-integrations/connectors/source-jira/source_jira/streams.py

@@ -373,6 +373,7 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg
             "parent",
             "priority",
             "project",
+            "resolutiondate",
             "security",
             "status",
             "subtasks",
@@ -530,6 +531,8 @@ class IssueProperties(StartDateJiraStream):
     https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-properties/#api-rest-api-3-issue-issueidorkey-properties-propertykey-get
     """
 
+    primary_key = "key"
+
     def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str:
         key = stream_slice["key"]
         issue_key = stream_slice["issue_key"]

airbyte-integrations/connectors/source-klaviyo/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=0.1.4
 LABEL io.airbyte.name=airbyte/source-klaviyo

airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml

@@ -13,6 +13,7 @@ tests:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
+      empty_streams: ['flows']
   incremental:
     - config_path: "secrets/config.json"
       future_state_path: "integration_tests/abnormal_state.json"

airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json

@@ -4,5 +4,8 @@
   },
   "global_exclusions": {
     "timestamp": "2120-10-10T00:00:00Z"
+  },
+  "flows": {
+    "created": "2120-10-10 00:00:00"
   }
 }

airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json

@@ -59,6 +59,21 @@
       "cursor_field": null,
       "destination_sync_mode": "append",
       "primary_key": null
+    },
+    {
+      "stream": {
+        "name": "flows",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": null,
+        "default_cursor_field": null,
+        "source_defined_primary_key": [["id"]],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
     }
   ]
 }

airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas.py

@@ -104,6 +104,19 @@ class Event(BaseSchemaModel):
     statistic_id: str
     event_properties: dict
     person: dict
+    flow_id: Optional[str]
+    campaign_id: Optional[str]
+    flow_message_id: Optional[str]
+
+
+class Flow(BaseSchemaModel):
+    id: str
+    name: str
+    status: str
+    created: datetime
+    updated: datetime
+    customer_filter: Optional[dict]
+    trigger: dict
 
 
 class GlobalExclusion(BaseSchemaModel):

airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py

@@ -10,7 +10,7 @@
 from airbyte_cdk.sources.streams import Stream
 from pydantic import Field
 from pydantic.main import BaseModel
-from source_klaviyo.streams import Campaigns, Events, GlobalExclusions, Lists, Metrics
+from source_klaviyo.streams import Campaigns, Events, Flows, GlobalExclusions, Lists, Metrics
 
 
 class ConnectorConfig(BaseModel):
@@ -61,6 +61,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
             GlobalExclusions(api_key=config.api_key, start_date=config.start_date),
             Lists(api_key=config.api_key),
             Metrics(api_key=config.api_key),
+            Flows(api_key=config.api_key, start_date=config.start_date),
         ]
 
     def spec(self, *args, **kwargs) -> ConnectorSpecification:

airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py

@@ -2,13 +2,14 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+import datetime
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
 
 import pendulum
 import requests
 from airbyte_cdk.sources.streams.http import HttpStream
-from source_klaviyo.schemas import Campaign, Event, GlobalExclusion, Metric, PersonList
+from source_klaviyo.schemas import Campaign, Event, Flow, GlobalExclusion, Metric, PersonList
 
 
 class KlaviyoStream(HttpStream, ABC):
@@ -99,7 +100,13 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
         the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental.
         """
         state_ts = int(current_stream_state.get(self.cursor_field, 0))
-        return {self.cursor_field: max(latest_record.get(self.cursor_field), state_ts)}
+        latest_record = latest_record.get(self.cursor_field)
+
+        if isinstance(latest_record, str):
+            latest_record = datetime.datetime.strptime(latest_record, "%Y-%m-%d %H:%M:%S")
+            latest_record = datetime.datetime.timestamp(latest_record)
+
+        return {self.cursor_field: max(latest_record, state_ts)}
 
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         """
@@ -240,3 +247,26 @@ class Events(IncrementalKlaviyoStream):
 
     def path(self, **kwargs) -> str:
         return "metrics/timeline"
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        """
+        :return an iterable containing each record in the response
+        """
+        response_json = response.json()
+        for record in response_json.get("data", []):
+            flow = record["event_properties"].get("$flow")
+            flow_message_id = record["event_properties"].get("$message")
+
+            record["flow_id"] = flow
+            record["flow_message_id"] = flow_message_id
+            record["campaign_id"] = flow_message_id if not flow else None
+
+            yield record
+
+
+class Flows(ReverseIncrementalKlaviyoStream):
+    schema = Flow
+    cursor_field = "created"
+
+    def path(self, **kwargs) -> str:
+        return "flows"

airbyte-integrations/connectors/source-mixpanel/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.1.15
+LABEL io.airbyte.version=0.1.16
 LABEL io.airbyte.name=airbyte/source-mixpanel