✨Source Stripe: New Streams - Prices (#26127)

* Initial commit

* Update metadata.yaml

* Fix md file

* resolve comments

* fix: format python files with flake

* fix: change created type in price schema to integer

* update expected_records of prices stream

---------

Co-authored-by: sajarin <sajarindider@gmail.com>

airbyte-integrations/connectors/source-stripe/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=3.16.0
+LABEL io.airbyte.version=3.17.0
 LABEL io.airbyte.name=airbyte/source-stripe

airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json

@@ -132,6 +132,13 @@
       "stream_descriptor": { "name": "payouts" }
     }
   },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": { "created": 10000000000 },
+      "stream_descriptor": { "name": "prices" }
+    }
+  },
   {
     "type": "STREAM",
     "stream": {

airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json

@@ -112,6 +112,90 @@
       "destination_sync_mode": "overwrite",
       "primary_key": [["id"]]
     },
+    {
+      "stream": {
+        "name": "invoices",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["created"],
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "payment_intents",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["created"],
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "payouts",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["created"],
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "plans",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["created"],
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "prices",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["created"],
+      "primary_key": [["id"]]
+    },
+    {
+      "stream": {
+        "name": "products",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["created"],
+      "primary_key": [["id"]]
+    },
     {
       "stream": {
         "name": "charges",

airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl

@@ -28,6 +28,9 @@
 {"stream": "payouts", "data": {"id": "po_1MXKoPEcXtiJtvvhwmqjvKoO", "object": "payout", "amount": 665880, "arrival_date": 1675382400, "automatic": false, "balance_transaction": "txn_1MXKoQEcXtiJtvvh65SHFZS6", "created": 1675413601, "currency": "usd", "description": "", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account"}, "emitted_at": 1689691102064}
 {"stream": "payouts", "data": {"id": "po_1MWHzjEcXtiJtvvhIdUQHhLq", "object": "payout", "amount": 154900, "arrival_date": 1675123200, "automatic": false, "balance_transaction": "txn_1MWHzjEcXtiJtvvhtydemd2Y", "created": 1675164443, "currency": "usd", "description": "Test", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": "issuing", "source_type": "issuing", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account"}, "emitted_at": 1689691102066}
 {"stream": "plans", "data": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "emitted_at": 1689691103696}
+{"stream": "prices", "data": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600.0, "unit_amount_decimal": "12600"}, "emitted_at": 1690480900454}
+{"stream": "prices", "data": {"id": "price_1MX364EcXtiJtvvh6jKcimNL", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 1700.0, "unit_amount_decimal": "1700"}, "emitted_at": 1690480900634}
+{"stream": "prices", "data": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000.0, "unit_amount_decimal": "2000"}, "emitted_at": 1690480900634}
 {"stream": "products", "data": {"id": "prod_KouQ5ez86yREmB", "object": "product", "active": true, "attributes": [], "created": 1640124902, "default_price": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "description": null, "images": [], "livemode": false, "metadata": {}, "name": "edgao-test-product", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1675345058, "url": null}, "emitted_at": 1689684235151}
 {"stream": "products", "data": {"id": "prod_NHcKselSHfKdfc", "object": "product", "active": true, "attributes": [], "created": 1675345504, "default_price": "price_1MX364EcXtiJtvvhE3WgTl4O", "description": "Test Product 1 description", "images": ["https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdjBOT09UaHRiNVl2WmJ6clNYRUlmcFFD00cCBRNHnV"], "livemode": false, "metadata": {}, "name": "Test Product 1", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10301000", "type": "service", "unit_label": null, "updated": 1675345505, "url": null}, "emitted_at": 1689684235408}
 {"stream": "products", "data": {"id": "prod_NCgx1XP2IFQyKF", "object": "product", "active": true, "attributes": [], "created": 1674209524, "default_price": null, "description": null, "images": [], "livemode": false, "metadata": {}, "name": "tu", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1674209524, "url": null}, "emitted_at": 1689684235411}

airbyte-integrations/connectors/source-stripe/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e094cb9a-26de-4645-8761-65c0c425d1de
-  dockerImageTag: 3.16.0
+  dockerImageTag: 3.17.0
   dockerRepository: airbyte/source-stripe
   githubIssueLabel: source-stripe
   icon: stripe.svg

airbyte-integrations/connectors/source-stripe/source_stripe/schemas/prices.json

@@ -0,0 +1,87 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Prices Schema",
+  "additionalProperties": true,
+  "type": ["null", "object"],
+  "properties": {
+    "id": {
+      "type": ["null", "string"]
+    },
+    "object": {
+      "type": ["null", "string"]
+    },
+    "active": {
+      "type": ["null", "boolean"]
+    },
+    "billing_scheme": {
+      "type": ["null", "string"]
+    },
+    "created": {
+      "type": ["null", "integer"]
+    },
+    "currency": {
+      "type": ["null", "string"]
+    },
+    "custom_unit_amount": {
+      "type": ["null", "string"]
+    },
+    "livemode": {
+      "type": ["null", "boolean"]
+    },
+    "lookup_key": {
+      "type": ["null", "string"]
+    },
+    "metadata": {
+      "type": ["null", "object"],
+      "properties": {
+        "nickname": {
+          "type": ["null", "string"]
+        }
+      }
+    },
+    "nickname": {
+      "type": ["null", "string"]
+    },
+    "product": {
+      "type": ["null", "string"]
+    },
+    "recurring": {
+      "type": ["null", "object"],
+      "properties": {
+        "aggregate_usage": {
+          "type": ["null", "string"]
+        },
+        "interval": {
+          "type": ["null", "string"]
+        },
+        "interval_count": {
+          "type": ["null", "number"]
+        },
+        "trial_period_days": {
+          "type": ["null", "string"]
+        },
+        "usage_type": {
+          "type": ["null", "string"]
+        }
+      }
+    },
+    "tax_behavior": {
+      "type": ["null", "string"]
+    },
+    "tiers_mode": {
+      "type": ["null", "string"]
+    },
+    "transform_quantity": {
+      "type": ["null", "string"]
+    },
+    "type": {
+      "type": ["null", "string"]
+    },
+    "unit_amount": {
+      "type": ["null", "number"]
+    },
+    "unit_amount_decimal": {
+      "type": ["null", "string"]
+    }
+  }
+}

airbyte-integrations/connectors/source-stripe/source_stripe/source.py

@@ -42,6 +42,7 @@
     Payouts,
     Persons,
     Plans,
+    Prices,
     Products,
     PromotionCodes,
     Refunds,
@@ -110,6 +111,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Payouts(**incremental_args),
             Persons(**incremental_args),
             Plans(**incremental_args),
+            Prices(**incremental_args),
             Products(**incremental_args),
             PromotionCodes(**incremental_args),
             Refunds(**incremental_args),

airbyte-integrations/connectors/source-stripe/source_stripe/streams.py

@@ -517,6 +517,17 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs):
         return params
 
 
+class Prices(IncrementalStripeStream):
+    """
+    API docs: https://stripe.com/docs/api/prices/list
+    """
+
+    cursor_field = "created"
+
+    def path(self, **kwargs):
+        return "prices"
+
+
 class Products(IncrementalStripeStream):
     """
     API docs: https://stripe.com/docs/api/products/list

airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py

@@ -42,7 +42,7 @@ def test_source_streams():
     with open("sample_files/config.json") as f:
         config = json.load(f)
     streams = SourceStripe().streams(config=config)
-    assert len(streams) == 45
+    assert len(streams) == 46
 
 
 @pytest.fixture(name="config")

airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py

@@ -29,6 +29,7 @@
     Payouts,
     Persons,
     Plans,
+    Prices,
     Products,
     PromotionCodes,
     Refunds,
@@ -176,6 +177,7 @@ def config_fixture():
         (Payouts, {}, "payouts"),
         (Persons, {"stream_slice": {"id": "A1"}}, "accounts/A1/persons"),
         (Plans, {}, "plans"),
+        (Prices, {}, "prices"),
         (Products, {}, "products"),
         (Subscriptions, {}, "subscriptions"),
         (SubscriptionItems, {}, "subscription_items"),

docs/integrations/sources/stripe.md

@@ -73,6 +73,7 @@ The Stripe source connector supports the following streams:
 - [Promotion Code](https://stripe.com/docs/api/promotion_codes/list) \(Incremental\)
 - [Persons](https://stripe.com/docs/api/persons/list) \(Incremental\)
 - [Plans](https://stripe.com/docs/api/plans/list) \(Incremental\)
+- [Prices](https://stripe.com/docs/api/prices/list) \(Incremental\)
 - [Products](https://stripe.com/docs/api/products/list) \(Incremental\)
 - [Refunds](https://stripe.com/docs/api/refunds/list) \(Incremental\)
 - [Reviews](https://stripe.com/docs/api/radar/reviews/list) \(Incremental\)
@@ -102,7 +103,8 @@ The Stripe connector should not run into Stripe API limitations under normal usa
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                              |
-|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 3.17.0  | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream                                                                                                                                  |
 | 3.16.0  | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas                                                                                                                     |
 | 3.15.0  | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams                                                                                                                             |
 | 3.14.0  | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream                                                                                                                           |