🎉 Source Shopify: update API version to 2022-10 (#18298)

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -983,7 +983,7 @@
 - name: Shopify
   sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
   dockerRepository: airbyte/source-shopify
-  dockerImageTag: 0.1.39
+  dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/shopify
   icon: shopify.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -10197,7 +10197,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-shopify:0.1.39"
+- dockerImage: "airbyte/source-shopify:0.2.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/shopify"
     connectionSpecification:

airbyte-integrations/connectors/source-shopify/Dockerfile

@@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.39
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/source-shopify

airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml

@@ -17,26 +17,21 @@ tests:
       status: "failed"
   discovery:
     - config_path: "secrets/config.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "0.1.38"
     - config_path: "secrets/config_old.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "0.1.38"
     - config_path: "secrets/config_oauth.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "0.1.38"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-      timeout_seconds: 3600
+      timeout_seconds: 7200
       # some streams hold data only for some time, therefore certain streams could be empty while sync.
       # 'abandoned_checkouts' stream holds data up to 1 month.
       empty_streams: ["abandoned_checkouts", "balance_transactions"]
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
+      timeout_seconds: 3600
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-      timeout_seconds: 3600
+      timeout_seconds: 7200

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "note_attributes": {
       "type": ["null", "array"],

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/balance_transactions.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": "integer"

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "commentable": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json

@@ -57,5 +57,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "last_order_name": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "assigned_location_id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "created_at": {
       "type": ["null", "string"],

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_items.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "active": {
       "type": ["null", "boolean"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_blogs.json

@@ -39,5 +39,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json

@@ -39,5 +39,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json

@@ -42,5 +42,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "order_id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "author": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "allocation_method": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "created_at": {
       "type": ["null", "string"],

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json

@@ -1,5 +1,6 @@
 {
   "type": ["object", "null"],
+  "additionalProperties": true,
   "properties": {
     "published_at": {
       "type": ["null", "string"],

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "address1": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json

@@ -1,5 +1,6 @@
 {
   "type": ["null", "object"],
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/tender_transactions.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": true,
   "properties": {
     "id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json

@@ -96,5 +96,6 @@
       "type": ["null", "string"]
     }
   },
+  "additionalProperties": true,
   "type": ["null", "object"]
 }

airbyte-integrations/connectors/source-shopify/source_shopify/source.py

@@ -22,7 +22,7 @@
 
 class ShopifyStream(HttpStream, ABC):
     # Latest Stable Release
-    api_version = "2021-07"
+    api_version = "2022-10"
     # Page size
     limit = 250
     # Define primary key as sort key for full_refresh, or very first sync for incremental_refresh
@@ -67,26 +67,27 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) ->
 
     @limiter.balance_rate_limit()
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        json_response = response.json() or {}
-        records = json_response.get(self.data_field, []) if self.data_field is not None else json_response
-        # transform method was implemented according to issue 4841
-        # Shopify API returns price fields as a string and it should be converted to number
-        # this solution designed to convert string into number, but in future can be modified for general purpose
-        if isinstance(records, dict):
-            # for cases when we have a single record as dict
-            # add shop_url to the record to make querying easy
-            records["shop_url"] = self.config["shop"]
-            yield self._transformer.transform(records)
-        else:
-            # for other cases
-            for record in records:
+        if response.status_code is requests.codes.OK:
+            json_response = response.json()
+            records = json_response.get(self.data_field, []) if self.data_field is not None else json_response
+            # transform method was implemented according to issue 4841
+            # Shopify API returns price fields as a string and it should be converted to number
+            # this solution designed to convert string into number, but in future can be modified for general purpose
+            if isinstance(records, dict):
+                # for cases when we have a single record as dict
                 # add shop_url to the record to make querying easy
-                record["shop_url"] = self.config["shop"]
-                yield self._transformer.transform(record)
+                records["shop_url"] = self.config["shop"]
+                yield self._transformer.transform(records)
+            else:
+                # for other cases
+                for record in records:
+                    # add shop_url to the record to make querying easy
+                    record["shop_url"] = self.config["shop"]
+                    yield self._transformer.transform(record)
 
     def should_retry(self, response: requests.Response) -> bool:
         if response.status_code == 404:
-            self.logger.warn(f"Stream `{self.name}` is not available, skipping.")
+            self.logger.warn(f"Stream `{self.name}` is not available, skipping...")
             setattr(self, "raise_on_http_errors", False)
             return False
         return super().should_retry(response)

docs/integrations/sources/shopify.md

@@ -145,6 +145,7 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro
 
 | Version | Date       | Pull Request                                              | Subject                                                                                                   |
 |:--------|:-----------|:----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------|
+| 0.2.0   | 2022-10-21 | [18298](https://github.com/airbytehq/airbyte/pull/18298)  | Updated API version to the `2022-10`, make stream schemas backward cpmpatible                                                                                 |
 | 0.1.39  | 2022-10-13 | [17962](https://github.com/airbytehq/airbyte/pull/17962)  | Add metafield streams; support for nested list streams                                                    |
 | 0.1.38  | 2022-10-10 | [17777](https://github.com/airbytehq/airbyte/pull/17777)  | Fixed `404` for configured streams, fix missing `cursor` error for old records                            |
 | 0.1.37  | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500)  | Improve input configuration copy                                                                          |