🎉 Source Notion: Adds pagination to Users stream

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -508,7 +508,7 @@
 - name: Notion
   sourceDefinitionId: 6e00b415-b02e-4160-bf02-58176a0ae687
   dockerRepository: airbyte/source-notion
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.io/integrations/sources/notion
   icon: notion.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -5542,7 +5542,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-notion:0.1.2"
+- dockerImage: "airbyte/source-notion:0.1.3"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/notion"
     connectionSpecification:

airbyte-integrations/connectors/source-notion/Dockerfile

@@ -34,5 +34,5 @@ COPY source_notion ./source_notion
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-notion

airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json

@@ -57,6 +57,7 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
+        "color": { "type": "string" },
         "text": { "type": "array", "items": { "$ref": "rich_text.json" } },
         "icon": { "$ref": "icon.json" },
         "children": { "type": "array", "items": { "type": "object" } }
@@ -75,6 +76,7 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
+        "color": { "type": "string" },
         "text": { "type": "array", "items": { "$ref": "rich_text.json" } },
         "language": {
           "enum": [

airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json

@@ -21,6 +21,15 @@
         "$ref": "rich_text.json"
       }
     },
+    "last_edited_by": {
+      "$ref": "user.json"
+    },
+    "created_by": {
+      "$ref": "user.json"
+    },
+    "archived": {
+      "type": "boolean"
+    },
     "icon": {
       "$ref": "icon.json"
     },

airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json

@@ -3,6 +3,7 @@
   "type": ["null", "object"],
   "additionalProperties": false,
   "properties": {
+    "color": { "type": "string" },
     "text": { "type": "array", "items": { "$ref": "rich_text.json" } }
   }
 }

airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json

@@ -3,6 +3,7 @@
   "type": "object",
   "additionalProperties": false,
   "properties": {
+    "color": { "type": "string" },
     "text": { "type": "array", "items": { "$ref": "rich_text.json" } },
     "children": { "type": "array", "items": { "type": "object" } }
   }

airbyte-integrations/connectors/source-notion/source_notion/streams.py

@@ -149,6 +149,12 @@ class Users(NotionStream):
     def path(self, **kwargs) -> str:
         return "users"
 
+    def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]:
+        params = {"page_size": self.page_size}
+        if next_page_token:
+            params["start_cursor"] = next_page_token["next_cursor"]
+        return params
+
 
 class Databases(IncrementalNotionStream):
     """

airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py

@@ -3,11 +3,13 @@
 #
 
 from http import HTTPStatus
+import random
 from unittest.mock import MagicMock
 
 import pytest
 import requests
-from source_notion.streams import NotionStream
+from airbyte_cdk.models import SyncMode
+from source_notion.streams import NotionStream, Users
 
 
 @pytest.fixture
@@ -76,3 +78,56 @@ def test_backoff_time(patch_base_class):
     stream = NotionStream(config=MagicMock())
     expected_backoff_time = None
     assert stream.backoff_time(response_mock) == expected_backoff_time
+
+
+def test_users_request_params(patch_base_class):
+    stream = Users(config=MagicMock())
+
+    # No next_page_token. First pull
+    inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None}
+    expected_params = {"page_size": 100}
+    assert stream.request_params(**inputs) == expected_params
+
+    # When getting pages after the first pull.
+    inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"next_cursor": "123"}}
+    expected_params = {"start_cursor": "123", "page_size": 100}
+    assert stream.request_params(**inputs) == expected_params
+
+
+def test_user_stream_handles_pagination_correclty(requests_mock):
+    """
+    Test shows that Users stream uses pagination as per Notion API docs.
+    """
+
+    response_body = {
+        "object": "list",
+        "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(100)],
+        "next_cursor": "bc48234b-77b2-41a6-95a3-6a8abb7887d5",
+        "has_more": True,
+        "type": "user"
+    }
+    requests_mock.get("https://api.notion.com/v1/users?page_size=100", json=response_body)
+
+    response_body = {
+        "object": "list",
+        "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(100, 200)],
+        "next_cursor": "67030467-b97b-4729-8fd6-2fb33d012da4",
+        "has_more": True,
+        "type": "user"
+    }
+    requests_mock.get("https://api.notion.com/v1/users?page_size=100&start_cursor=bc48234b-77b2-41a6-95a3-6a8abb7887d5", json=response_body)
+
+    response_body = {
+        "object": "list",
+        "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(200, 220)],
+        "next_cursor": None,
+        "has_more": False,
+        "type": "user"
+    }
+    requests_mock.get("https://api.notion.com/v1/users?page_size=100&start_cursor=67030467-b97b-4729-8fd6-2fb33d012da4", json=response_body)
+
+    stream = Users(config=MagicMock())
+
+    records = stream.read_records(sync_mode=SyncMode.full_refresh)
+    records_length = sum(1 for _ in records)
+    assert records_length == 220

docs/integrations/sources/notion.md

@@ -57,6 +57,7 @@ Please register on Notion and follow this [docs](https://developers.notion.com/d
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream |
 | 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL |
 | 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description |
 | 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release |