✨ Source Mixpanel: add page size to configuration to increase sync speed

airbyte-integrations/connectors/source-mixpanel/metadata.yaml

@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a
-  dockerImageTag: 3.2.4
+  dockerImageTag: 3.3.0
   dockerRepository: airbyte/source-mixpanel
   documentationUrl: https://docs.airbyte.com/integrations/sources/mixpanel
   githubIssueLabel: source-mixpanel

airbyte-integrations/connectors/source-mixpanel/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.2.4"
+version = "3.3.0"
 name = "source-mixpanel"
 description = "Source implementation for Mixpanel."
 authors = ["Airbyte <contact@airbyte.io>"]

airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py

@@ -291,7 +291,7 @@ def next_page_token(self, response, last_records: List[Mapping[str, Any]]) -> Op
         if total:
             self._total = total
 
-        if self._total and page_number is not None and self._total > self.page_size * (page_number + 1):
+        if self._total and page_number is not None and self._total > self._page_size * (page_number + 1):
             return {"session_id": decoded_response.get("session_id"), "page": page_number + 1}
         else:
             self._total = None

airbyte-integrations/connectors/source-mixpanel/source_mixpanel/manifest.yaml

@@ -44,6 +44,9 @@ definitions:
       - error_message_contains: "Query rate limit exceeded"
         action: RETRY
         error_message: Query rate limit exceeded.
+      - error_message_contains: "unknown error"
+        action: RETRY
+        error_message: An unknown error occurred
       - error_message_contains: "to_date cannot be later than today"
         action: FAIL
         error_message: Your project timezone must be misconfigured. Please set it to the one defined in your Mixpanel project settings.
@@ -150,7 +153,7 @@ definitions:
       type: CustomPaginationStrategy
       class_name: "source_mixpanel.components.EngagePaginationStrategy"
       start_from_page: 1
-      page_size: 1000
+      page_size: "{{ config.page_size }}"
     page_token_option:
       type: RequestOption
       inject_into: request_parameter

airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py

@@ -72,7 +72,17 @@ def validate_date(name: str, date_str: str, default: pendulum.date) -> pendulum.
 
     @adapt_validate_if_testing
     def _validate_and_transform(self, config: MutableMapping[str, Any]):
-        project_timezone, start_date, end_date, attribution_window, select_properties_by_default, region, date_window_size, project_id = (
+        (
+            project_timezone,
+            start_date,
+            end_date,
+            attribution_window,
+            select_properties_by_default,
+            region,
+            date_window_size,
+            project_id,
+            page_size,
+        ) = (
             config.get("project_timezone", "US/Pacific"),
             config.get("start_date"),
             config.get("end_date"),
@@ -81,6 +91,7 @@ def _validate_and_transform(self, config: MutableMapping[str, Any]):
             config.get("region", "US"),
             config.get("date_window_size", 30),
             config.get("credentials", dict()).get("project_id"),
+            config.get("page_size", 1000),
         )
         try:
             project_timezone = pendulum.timezone(project_timezone)
@@ -113,5 +124,6 @@ def _validate_and_transform(self, config: MutableMapping[str, Any]):
         config["region"] = region
         config["date_window_size"] = date_window_size
         config["project_id"] = project_id
+        config["page_size"] = page_size
 
         return config

airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json

@@ -119,6 +119,14 @@
         "type": "integer",
         "minimum": 1,
         "default": 30
+      },
+      "page_size": {
+        "order": 9,
+        "title": "Page Size",
+        "description": "The number of records to fetch per request. Default is 1000.",
+        "type": "integer",
+        "minimum": 1,
+        "default": 1000
       }
     }
   }

airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py

@@ -23,6 +23,7 @@ def config(start_date):
         "start_date": start_date,
         "end_date": start_date.add(days=31),
         "region": "US",
+        "page_size": 1000,
     }
 
 

airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py

@@ -94,37 +94,37 @@ def test_streams_string_date(requests_mock, config_raw):
     "config, success, expected_error_message",
     (
         (
-            {"credentials": {"api_secret": "secret"}, "project_timezone": "Miami"},
+            {"credentials": {"api_secret": "secret"}, "project_timezone": "Miami", "page_size": 1000},
             False,
             "Could not parse time zone: Miami, please enter a valid timezone.",
         ),
         (
-            {"credentials": {"api_secret": "secret"}, "start_date": "20 Jan 2021"},
+            {"credentials": {"api_secret": "secret"}, "start_date": "20 Jan 2021", "page_size": 1000},
             False,
             "Could not parse start date: 20 Jan 2021. Please enter a valid start date.",
         ),
         (
-            {"credentials": {"api_secret": "secret"}, "end_date": "20 Jan 2021"},
+            {"credentials": {"api_secret": "secret"}, "end_date": "20 Jan 2021", "page_size": 1000},
             False,
             "Could not parse end date: 20 Jan 2021. Please enter a valid end date.",
         ),
         (
-            {"credentials": {"api_secret": "secret"}, "attribution_window": "20 days"},
+            {"credentials": {"api_secret": "secret"}, "attribution_window": "20 days", "page_size": 1000},
             False,
             "Please provide a valid integer for the `Attribution window` parameter.",
         ),
         (
-            {"credentials": {"api_secret": "secret"}, "select_properties_by_default": "Yes"},
+            {"credentials": {"api_secret": "secret"}, "select_properties_by_default": "Yes", "page_size": 1000},
             False,
             "Please provide a valid True/False value for the `Select properties by default` parameter.",
         ),
-        ({"credentials": {"api_secret": "secret"}, "region": "UK"}, False, "Region must be either EU or US."),
+        ({"credentials": {"api_secret": "secret"}, "region": "UK", "page_size": 1000}, False, "Region must be either EU or US."),
         (
-            {"credentials": {"username": "user", "secret": "secret"}},
+            {"credentials": {"username": "user", "secret": "secret"}, "page_size": 1000},
             False,
             "Required parameter 'project_id' missing or malformed. Please provide a valid project ID.",
         ),
-        ({"credentials": {"api_secret": "secret"}, "region": "EU", "start_date": "2021-02-01T00:00:00Z"}, True, None),
+        ({"credentials": {"api_secret": "secret"}, "region": "EU", "start_date": "2021-02-01T00:00:00Z", "page_size": 1000}, True, None),
         (
             {
                 "credentials": {"username": "user", "secret": "secret", "project_id": 2397709},
@@ -135,6 +135,7 @@ def test_streams_string_date(requests_mock, config_raw):
                 "select_properties_by_default": True,
                 "region": "EU",
                 "date_window_size": 10,
+                "page_size": 1000
             },
             True,
             None,

docs/integrations/sources/mixpanel.md

@@ -58,6 +58,7 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                                                                                                                                                            |
 |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 3.3.0 | 2024-07-15 | [41754](https://github.com/airbytehq/airbyte/pull/41754) | Add engage page size to configuration |
 | 3.2.4 | 2024-07-13 | [41754](https://github.com/airbytehq/airbyte/pull/41754) | Update dependencies |
 | 3.2.3 | 2024-07-10 | [41420](https://github.com/airbytehq/airbyte/pull/41420) | Update dependencies |
 | 3.2.2 | 2024-07-09 | [41289](https://github.com/airbytehq/airbyte/pull/41289) | Update dependencies |