From 5e302e0a8a9a47a95dd9e80841ff075ea926d55d Mon Sep 17 00:00:00 2001 From: Baz Date: Mon, 24 Oct 2022 15:23:15 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Shopify:=20update=20API?= =?UTF-8?q?=20version=20to=20`2022-10`=20(#18298)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-shopify/Dockerfile | 2 +- .../source-shopify/acceptance-test-config.yml | 11 ++---- .../schemas/abandoned_checkouts.json | 1 + .../source_shopify/schemas/articles.json | 1 + .../schemas/balance_transactions.json | 1 + .../source_shopify/schemas/blogs.json | 1 + .../source_shopify/schemas/collections.json | 1 + .../source_shopify/schemas/collects.json | 1 + .../schemas/custom_collections.json | 1 + .../source_shopify/schemas/customers.json | 1 + .../schemas/discount_codes.json | 1 + .../source_shopify/schemas/draft_orders.json | 1 + .../schemas/fulfillment_orders.json | 1 + .../source_shopify/schemas/fulfillments.json | 1 + .../schemas/inventory_items.json | 1 + .../schemas/inventory_levels.json | 1 + .../source_shopify/schemas/locations.json | 1 + .../schemas/metafield_articles.json | 1 + .../schemas/metafield_blogs.json | 1 + .../schemas/metafield_collections.json | 1 + .../schemas/metafield_customers.json | 1 + .../schemas/metafield_draft_orders.json | 1 + .../schemas/metafield_locations.json | 1 + .../schemas/metafield_orders.json | 1 + .../schemas/metafield_pages.json | 1 + .../schemas/metafield_product_images.json | 1 + .../schemas/metafield_product_variants.json | 1 + .../schemas/metafield_products.json | 1 + .../schemas/metafield_shops.json | 1 + .../schemas/metafield_smart_collections.json | 1 + .../source_shopify/schemas/order_refunds.json | 1 + .../source_shopify/schemas/order_risks.json | 1 + .../source_shopify/schemas/orders.json | 1 + .../source_shopify/schemas/pages.json | 1 + .../source_shopify/schemas/price_rules.json | 1 + .../schemas/product_images.json | 1 + .../schemas/product_variants.json | 1 + .../source_shopify/schemas/products.json | 1 + .../source_shopify/schemas/shop.json | 1 + .../schemas/smart_collections.json | 1 + .../schemas/tender_transactions.json | 1 + .../source_shopify/schemas/transactions.json | 1 + .../source-shopify/source_shopify/source.py | 35 ++++++++++--------- docs/integrations/sources/shopify.md | 1 + 46 files changed, 65 insertions(+), 28 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index de0d83896049..4dee616385ea 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -983,7 +983,7 @@ - name: Shopify sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 dockerRepository: airbyte/source-shopify - dockerImageTag: 0.1.39 + dockerImageTag: 0.2.0 documentationUrl: https://docs.airbyte.com/integrations/sources/shopify icon: shopify.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e3f0004fddf3..0aeba3f5630f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10197,7 +10197,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-shopify:0.1.39" +- dockerImage: "airbyte/source-shopify:0.2.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/shopify" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile index 62bbff5455f6..5925cb39c485 100644 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.39 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index 5c542bfe96e0..af7a2ec30b0c 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -17,18 +17,12 @@ tests: status: "failed" discovery: - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.38" - config_path: "secrets/config_old.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.38" - config_path: "secrets/config_oauth.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.38" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 7200 # some streams hold data only for some time, therefore certain streams could be empty while sync. # 'abandoned_checkouts' stream holds data up to 1 month. empty_streams: ["abandoned_checkouts", "balance_transactions"] @@ -36,7 +30,8 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 3600 full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 7200 diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json index f9be732c12d5..6c9d434e6462 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "note_attributes": { "type": ["null", "array"], diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json index f1b09743f014..85ec2388ea75 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/balance_transactions.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/balance_transactions.json index 25fc3606f6ac..6ada82c01327 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/balance_transactions.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/balance_transactions.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json index 003a867b5672..a9ea23eea82e 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "commentable": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json index 6243a695693e..bd34860778a7 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collections.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json index 05495af8ce4d..30b93e0e7ca0 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json index a52a84d3ed67..85041b590019 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json @@ -57,5 +57,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json index b7ef5f06b574..6e3b3b41d756 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "last_order_name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json index 7f5f0d425bca..f3e208e014c5 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json index 3c30979cfb68..d08ac15f22da 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json index b4ae5dc07d09..9d3a417eefe5 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "assigned_location_id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json index c106c0b50f76..8822ba1be0cc 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "created_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_items.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_items.json index 908bf2a21b50..bba96062be2d 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_items.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_items.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json index 729bbbc28e0b..6a0ce92a52c7 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/inventory_levels.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json index a25575e2a3bd..50dde9482014 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/locations.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "active": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_articles.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_blogs.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_blogs.json index 3a4a7bc1fcbc..d90da9f20257 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_blogs.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_blogs.json @@ -39,5 +39,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json index 3a4a7bc1fcbc..d90da9f20257 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_collections.json @@ -39,5 +39,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_customers.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_draft_orders.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_locations.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_orders.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_pages.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_images.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_product_variants.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_products.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_shops.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json index 1e91c726368f..b83ed8ea6279 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafield_smart_collections.json @@ -42,5 +42,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json index dbb0b7fbf1ea..764854896123 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "order_id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json index 04d364c93f80..ea6f5a04b671 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json index 18da58fcc2c2..7100f8be9934 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json index 95de888f2080..444ae832bc8f 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "author": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json index b04a19349cd0..eb95deb9da9e 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "allocation_method": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json index 378cb2463ca8..8463b07565b6 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_images.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "created_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json index 9b958269e78c..1a9554c48751 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json index ca403f562201..90066b9aefeb 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json @@ -1,5 +1,6 @@ { "type": ["object", "null"], + "additionalProperties": true, "properties": { "published_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json index d29981f47486..9a41fd623e13 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "address1": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json index a931b2c72b76..8289d6d03ba5 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/smart_collections.json @@ -1,5 +1,6 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/tender_transactions.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/tender_transactions.json index 5fce094cc681..2e52c04a203a 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/tender_transactions.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/tender_transactions.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json index a3410fcf8a70..22590970d6cf 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json @@ -96,5 +96,6 @@ "type": ["null", "string"] } }, + "additionalProperties": true, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index 700d9113bd1b..9f32c1209d49 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -22,7 +22,7 @@ class ShopifyStream(HttpStream, ABC): # Latest Stable Release - api_version = "2021-07" + api_version = "2022-10" # Page size limit = 250 # Define primary key as sort key for full_refresh, or very first sync for incremental_refresh @@ -67,26 +67,27 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> @limiter.balance_rate_limit() def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() or {} - records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - # transform method was implemented according to issue 4841 - # Shopify API returns price fields as a string and it should be converted to number - # this solution designed to convert string into number, but in future can be modified for general purpose - if isinstance(records, dict): - # for cases when we have a single record as dict - # add shop_url to the record to make querying easy - records["shop_url"] = self.config["shop"] - yield self._transformer.transform(records) - else: - # for other cases - for record in records: + if response.status_code is requests.codes.OK: + json_response = response.json() + records = json_response.get(self.data_field, []) if self.data_field is not None else json_response + # transform method was implemented according to issue 4841 + # Shopify API returns price fields as a string and it should be converted to number + # this solution designed to convert string into number, but in future can be modified for general purpose + if isinstance(records, dict): + # for cases when we have a single record as dict # add shop_url to the record to make querying easy - record["shop_url"] = self.config["shop"] - yield self._transformer.transform(record) + records["shop_url"] = self.config["shop"] + yield self._transformer.transform(records) + else: + # for other cases + for record in records: + # add shop_url to the record to make querying easy + record["shop_url"] = self.config["shop"] + yield self._transformer.transform(record) def should_retry(self, response: requests.Response) -> bool: if response.status_code == 404: - self.logger.warn(f"Stream `{self.name}` is not available, skipping.") + self.logger.warn(f"Stream `{self.name}` is not available, skipping...") setattr(self, "raise_on_http_errors", False) return False return super().should_retry(response) diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index da8df47995ec..c8bd98223f2a 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -145,6 +145,7 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 0.2.0 | 2022-10-21 | [18298](https://github.com/airbytehq/airbyte/pull/18298) | Updated API version to the `2022-10`, make stream schemas backward cpmpatible | | 0.1.39 | 2022-10-13 | [17962](https://github.com/airbytehq/airbyte/pull/17962) | Add metafield streams; support for nested list streams | | 0.1.38 | 2022-10-10 | [17777](https://github.com/airbytehq/airbyte/pull/17777) | Fixed `404` for configured streams, fix missing `cursor` error for old records | | 0.1.37 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy |