🐛 Source SurveyMonkey: fix add missing params to stream survey_respon…

…ses (#25109) * add missing params to stream survey_responses * fix unit test test_surveys_responses_request_params * use next_page_token when given in survey_responses stream * fix integration tests * fix build * update expected records * update expected records complete * fix expected records * bump connector version and update docs * Update source_definitions.yaml * auto-bump connector version --------- Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Apr 27, 2023 · f4d2c64 · f4d2c64
1 parent fe5e438
commit f4d2c64
Show file tree

Hide file tree

Showing 10 changed files with 49 additions and 39 deletions.
diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json
@@ -24906,7 +24906,7 @@
     "sourceDefinitionId": "badc5925-0485-42be-8caa-b34096cb71b5",
     "name": "SurveyMonkey",
     "dockerRepository": "airbyte/source-surveymonkey",
-    "dockerImageTag": "0.2.0",
+    "dockerImageTag": "0.2.1",
     "documentationUrl": "https://docs.airbyte.com/integrations/sources/surveymonkey",
     "icon": "surveymonkey.svg",
     "sourceType": "api",

diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml
@@ -2044,7 +2044,7 @@
 - name: SurveyMonkey
   sourceDefinitionId: badc5925-0485-42be-8caa-b34096cb71b5
   dockerRepository: airbyte/source-surveymonkey
-  dockerImageTag: 0.2.0
+  dockerImageTag: 0.2.1
   documentationUrl: https://docs.airbyte.com/integrations/sources/surveymonkey
   icon: surveymonkey.svg
   sourceType: api

diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml
@@ -15295,7 +15295,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-surveymonkey:0.2.0"
+- dockerImage: "airbyte/source-surveymonkey:0.2.1"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/surveymonkey"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile
@@ -12,5 +12,6 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.0
+
+LABEL io.airbyte.version=0.2.1
 LABEL io.airbyte.name=airbyte/source-surveymonkey
diff --git a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml
@@ -3,7 +3,7 @@ test_strictness_level: high
 acceptance_tests:
   spec:
     tests:
-    - spec_path: "source_surveymonkey/spec.json"
+      - spec_path: "source_surveymonkey/spec.json"
   connection:
     tests:
       - config_path: "secrets/config_old.json"
@@ -14,20 +14,20 @@ acceptance_tests:
         status: "failed"
   discovery:
     tests:
-    - config_path: "secrets/config.json"
+      - config_path: "secrets/config.json"
   basic_read:
     tests:
-    - config_path: "secrets/config.json"
-      expect_records:
-        path: "integration_tests/expected_records.txt"
-      fail_on_extra_columns: false
+      - config_path: "secrets/config.json"
+        expect_records:
+          path: "integration_tests/expected_records.txt"
+        fail_on_extra_columns: false
   incremental:
     tests:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      future_state:
-        future_state_path: "integration_tests/abnormal_state.json"
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     tests:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/expected_records.txt
diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py
@@ -110,7 +110,6 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str,
 
 
 class IncrementalSurveymonkeyStream(SurveymonkeyStream, ABC):
-
     state_checkpoint_interval = 1000
 
     @property
@@ -128,7 +127,6 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
 
 
 class SurveyIds(IncrementalSurveymonkeyStream):
-
     cursor_field = "date_modified"
 
     def path(self, **kwargs) -> str:
@@ -137,7 +135,7 @@ def path(self, **kwargs) -> str:
     def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]:
         params = super().request_params(stream_state=stream_state, **kwargs)
         params["sort_order"] = "ASC"
-        params["sort_by"] = "date_modified"
+        params["sort_by"] = self.cursor_field
         params["per_page"] = 1000  # maybe as user input or bigger value
         since_value = pendulum.parse(stream_state.get(self.cursor_field)) if stream_state.get(self.cursor_field) else self._start_date
 
@@ -219,7 +217,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str,
 
 class SurveyResponses(SurveyIDSliceMixin, IncrementalSurveymonkeyStream):
     """
-    Docs: https://developer.surveymonkey.com/api/v3/#survey-responses
+    Docs: https://developer.surveymonkey.com/api/v3/#api-endpoints-survey-responses
     """
 
     cursor_field = "date_modified"
@@ -249,8 +247,18 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
         current_stream_state[survey_id] = {self.cursor_field: state_value}
         return current_stream_state
 
-    def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]:
-        params = super().request_params(stream_state=stream_state, **kwargs)
+    def request_params(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> MutableMapping[str, Any]:
+        if next_page_token:
+            return next_page_token
+
+        params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token)
+        params["sort_order"] = "ASC"
+        params["sort_by"] = self.cursor_field
+        # Max of 100 allowed per page. We use the highest
+        # possible value to reduce the number of API calls.
+        params["per_page"] = 100
 
         since_value_surv = stream_state.get(stream_slice["survey_id"])
         if since_value_surv:

diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py
@@ -396,11 +396,11 @@ def test_surveys_responses_get_updated_state(current_stream_state, latest_record
     [
         (
             {"307785415": {"date_modified": "2021-01-01T00:00:00+00:00"}},
-            {"start_modified_at": "2021-01-01T00:00:00"},
+            {"sort_order": "ASC", "sort_by": "date_modified", "per_page": 100, "start_modified_at": "2021-01-01T00:00:00"},
         ),
         (
             {},
-            {"start_modified_at": "2000-01-01T00:00:00"},  # return start_date
+            {"sort_order": "ASC", "sort_by": "date_modified", "per_page": 100, "start_modified_at": "2000-01-01T00:00:00"},  # return start_date
         ),
     ],
 )

diff --git a/connectors.md b/connectors.md
@@ -233,7 +233,7 @@
 | **Strava** | <img alt="Strava icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/strava.svg" height="30" height="30"/> | Source | airbyte/source-strava:0.1.4 | beta | [docs](https://docs.airbyte.com/integrations/sources/strava) | [connectors/source/strava](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/strava) | [source-strava](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-strava) | <small>`7a4327c4-315a-11ec-8d3d-0242ac130003`</small> |
 | **Stripe** | <img alt="Stripe icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/stripe.svg" height="30" height="30"/> | Source | airbyte/source-stripe:3.4.1 | generally_available | [docs](https://docs.airbyte.com/integrations/sources/stripe) | [connectors/source/stripe](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/stripe) | [source-stripe](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-stripe) | <small>`e094cb9a-26de-4645-8761-65c0c425d1de`</small> |
 | **SurveyCTO** | <img alt="SurveyCTO icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/surveycto.svg" height="30" height="30"/> | Source | airbyte/source-surveycto:0.1.1 | alpha | [docs](https://docs.airbyte.com/integrations/sources/surveycto) | [connectors/source/surveycto](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/surveycto) | [source-surveycto](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-surveycto) | <small>`dd4632f4-15e0-4649-9b71-41719fb1fdee`</small> |
-| **SurveyMonkey** | <img alt="SurveyMonkey icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/surveymonkey.svg" height="30" height="30"/> | Source | airbyte/source-surveymonkey:0.2.0 | generally_available | [docs](https://docs.airbyte.com/integrations/sources/surveymonkey) | [connectors/source/surveymonkey](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/surveymonkey) | [source-surveymonkey](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-surveymonkey) | <small>`badc5925-0485-42be-8caa-b34096cb71b5`</small> |
+| **SurveyMonkey** | <img alt="SurveyMonkey icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/surveymonkey.svg" height="30" height="30"/> | Source | airbyte/source-surveymonkey:0.2.1 | generally_available | [docs](https://docs.airbyte.com/integrations/sources/surveymonkey) | [connectors/source/surveymonkey](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/surveymonkey) | [source-surveymonkey](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-surveymonkey) | <small>`badc5925-0485-42be-8caa-b34096cb71b5`</small> |
 | **SurveySparrow** | <img alt="SurveySparrow icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/surveysparrow.svg" height="30" height="30"/> | Source | airbyte/source-survey-sparrow:0.2.0 | alpha | [docs](https://docs.airbyte.com/integrations/sources/survey-sparrow) | [connectors/source/survey-sparrow](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/survey-sparrow) | [source-survey-sparrow](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-survey-sparrow) | <small>`4a4d887b-0f2d-4b33-ab7f-9b01b9072804`</small> |
 | **TMDb** | <img alt="TMDb icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/tmdb.svg" height="30" height="30"/> | Source | airbyte/source-tmdb:0.1.0 | alpha | [docs](https://docs.airbyte.com/integrations/sources/tmdb) | [connectors/source/tmdb](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/tmdb) | [source-tmdb](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tmdb) | <small>`6240848f-f795-45eb-8f5e-c7542822fc03`</small> |
 | **TPLcentral** | x | Source | airbyte/source-tplcentral:0.1.1 | alpha | [docs](https://docs.airbyte.com/integrations/sources/tplcentral) | [connectors/source/tplcentral](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/tplcentral) | [source-tplcentral](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tplcentral) | <small>`f9b6c538-ee12-42fe-8d4b-0c10f5955417`</small> |

diff --git a/docs/integrations/sources/surveymonkey.md b/docs/integrations/sources/surveymonkey.md
@@ -66,6 +66,7 @@ To cover more data from this source we use caching.
 
 | Version | Date       | Pull Request                                             | Subject                                                                |
 |:--------| :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------- |
+| 0.2.1   | 2023-04-27 | [25109](https://github.com/airbytehq/airbyte/pull/25109) | Fix add missing params to stream `SurveyResponses`                                |
 | 0.2.0   | 2023-04-18 | [23721](https://github.com/airbytehq/airbyte/pull/23721) |  Add `SurveyCollectors` and `Collectors` stream                                              |
 | 0.1.16  | 2023-04-13 | [25080](https://github.com/airbytehq/airbyte/pull/25080) |  Fix spec.json required fields and update schema for surveys and survey_responses                    |
 | 0.1.15  | 2023-02-11 | [22865](https://github.com/airbytehq/airbyte/pull/22865) | Specified date formatting in specification                                                     |