Source Stripe: implement slicing (#15292)

* #45 oncall - source Stripe: implement slicing * #45 source stripe: upd changelog * #45 source stripe: upd changelog * #45 source stripe: make slice range configurable * #45 source stripe: move generating a single slice into a mixin * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Aug 11, 2022 · 16d8133 · 16d8133
1 parent 167aa13
commit 16d8133
Show file tree

Hide file tree

Showing 11 changed files with 193 additions and 116 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -944,7 +944,7 @@
 - name: Stripe
   sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de
   dockerRepository: airbyte/source-stripe
-  dockerImageTag: 0.1.35
+  dockerImageTag: 0.1.36
   documentationUrl: https://docs.airbyte.io/integrations/sources/stripe
   icon: stripe.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -9345,7 +9345,7 @@
               type: "string"
               path_in_connector_config:
               - "client_secret"
-- dockerImage: "airbyte/source-stripe:0.1.35"
+- dockerImage: "airbyte/source-stripe:0.1.36"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/stripe"
     connectionSpecification:
@@ -9389,6 +9389,23 @@
             \ is frequently updated after creation. More info <a href=\"https://docs.airbyte.com/integrations/sources/stripe#requirements\"\
             >here</a>"
           order: 3
+        slice_range:
+          type: "integer"
+          title: "Data request time increment in days (Optional)"
+          default: 365
+          minimum: 1
+          examples:
+          - 1
+          - 3
+          - 10
+          - 30
+          - 180
+          - 360
+          description: "The time increment used by the connector when requesting data\
+            \ from the Stripe API. The bigger the value is,  the less requests will\
+            \ be made and faster the sync will be. On the other hand, the more seldom\
+            \  the state is persisted."
+          order: 4
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile
@@ -4,13 +4,13 @@ FROM python:3.9-slim
 RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /airbyte/integration_code
-COPY source_stripe ./source_stripe
-COPY main.py ./
 COPY setup.py ./
 RUN pip install .
+COPY source_stripe ./source_stripe
+COPY main.py ./
 
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.35
+LABEL io.airbyte.version=0.1.36
 LABEL io.airbyte.name=airbyte/source-stripe
diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json
@@ -1,20 +1,20 @@
 {
-  "charges": { "created": 161703040300 },
-  "coupons": { "created": 161703040300 },
-  "events": { "created": 161749384700 },
-  "customers": { "created": 160083796900 },
-  "plans": { "created": 159484835000 },
-  "invoices": { "created": 161749017500 },
-  "invoice_items": { "date": 159494698100 },
-  "transfers": { "created": 161099582400 },
-  "subscriptions": { "created": 159968687300 },
-  "balance_transactions": { "created": 161706755600 },
-  "payouts": { "created": 161706755600 },
-  "disputes": { "created": 161099630500 },
-  "products": { "created": 158551134100 },
-  "refunds": { "created": 161959562900 },
-  "payment_intents": { "created": 161959562900 },
-  "promotion_codes": { "created": 163534157100 },
+  "charges": { "created": 10000000000 },
+  "coupons": { "created": 10000000000 },
+  "events": { "created": 10000000000 },
+  "customers": { "created": 10000000000 },
+  "plans": { "created": 10000000000 },
+  "invoices": { "created": 10000000000 },
+  "invoice_items": { "date": 10000000000 },
+  "transfers": { "created": 10000000000 },
+  "subscriptions": { "created": 10000000000 },
+  "balance_transactions": { "created": 10000000000 },
+  "payouts": { "created": 10000000000 },
+  "disputes": { "created": 10000000000 },
+  "products": { "created": 10000000000 },
+  "refunds": { "created": 10000000000 },
+  "payment_intents": { "created": 10000000000 },
+  "promotion_codes": { "created": 10000000000 },
   "checkout_sessions": { "expires_at": 10000000000 },
   "checkout_sessions_line_items": { "checkout_session_expires_at": 10000000000 }
 }
diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/test_dummy.py b/airbyte-integrations/connectors/source-stripe/integration_tests/test_dummy.py
diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "stripe==2.56.0", "pendulum==1.2.0"]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "stripe==2.56.0", "pendulum==2.1.2"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",

diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py
@@ -51,7 +51,12 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         authenticator = TokenAuthenticator(config["client_secret"])
         start_date = pendulum.parse(config["start_date"]).int_timestamp
-        args = {"authenticator": authenticator, "account_id": config["account_id"], "start_date": start_date}
+        args = {
+            "authenticator": authenticator,
+            "account_id": config["account_id"],
+            "start_date": start_date,
+            "slice_range": config.get("slice_range"),
+        }
         incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")}
         return [
             BalanceTransactions(**incremental_args),

diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml
@@ -44,3 +44,14 @@ connectionSpecification:
         after creation. More info <a
         href="https://docs.airbyte.com/integrations/sources/stripe#requirements">here</a>
       order: 3
+    slice_range:
+      type: integer
+      title: Data request time increment in days (Optional)
+      default: 365
+      minimum: 1
+      examples: [1, 3, 10, 30, 180, 360]
+      description: >-
+        The time increment used by the connector when requesting data from the Stripe API. The bigger the value is, 
+        the less requests will be made and faster the sync will be. On the other hand, the more seldom 
+        the state is persisted.
+      order: 4