Skip to content

Commit

Permalink
🎉 Source Shopify: Use latest CDK version (#40593)
Browse files Browse the repository at this point in the history 
Co-authored-by: maxi297 <maxime@airbyte.io>
  • Loading branch information
@bazarnov @maxi297
bazarnov and maxi297 authored Jul 1, 2024
1 parent 554d45a commit 42c68d1
Show file tree
Hide file tree
Showing 9 changed files with 193 additions and 112 deletions.
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-shopify/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
dockerImageTag: 2.4.6
dockerImageTag: 2.4.7
dockerRepository: airbyte/source-shopify
documentationUrl: https://docs.airbyte.com/integrations/sources/shopify
githubIssueLabel: source-shopify
Expand Down
194 changes: 128 additions & 66 deletions airbyte-integrations/connectors/source-shopify/poetry.lock
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions airbyte-integrations/connectors/source-shopify/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
build-backend = "poetry.core.masonry.api"

[tool.poetry]
version = "2.4.6"
version = "2.4.7"
name = "source-shopify"
description = "Source CDK implementation for Shopify."
authors = [ "Airbyte <contact@airbyte.io>",]
Expand All @@ -17,9 +17,9 @@ include = "source_shopify"

[tool.poetry.dependencies]
python = "^3.9,<3.12"
airbyte-cdk = "^1"
airbyte-cdk = "^2"
sgqlc = "==16.3"
graphql-query = "^1.1.1"
graphql-query = "^1"

[tool.poetry.scripts]
source-shopify = "source_shopify.run:run"
Expand Down
72 changes: 41 additions & 31 deletions airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1203,23 +1203,29 @@ class InventoryLevel(ShopifyBulkQuery):
node {
__typename
id
inventoryLevels(query: "updated_at:>='2023-04-14T00:00:00+00:00'") {
inventoryLevels(query: "updated_at:>='2020-04-13T00:00:00+00:00'") {
edges {
node {
__typename
id
available
item {
inventory_item_id: id
inventoryHistoryUrl
locationsCount {
count
}
__typename
id
canDeactivate
createdAt
deactivationAlert
updatedAt
item {
inventory_history_url: inventoryHistoryUrl
inventory_item_id: id
locations_count: locationsCount {
count
}
}
quantities(
names: ["available", "incoming", "committed", "damaged", "on_hand", "quality_control", "reserved", "safety_stock"]) {
id
name
quantity
updatedAt
}
updatedAt
canDeactivate
createdAt
deactivationAlert
}
}
}
Expand Down Expand Up @@ -1251,13 +1257,6 @@ class InventoryLevel(ShopifyBulkQuery):
'"reserved"',
'"safety_stock"',
]
# quantities fields
quantities_fields: List[str] = [
"id",
"name",
"quantity",
"updatedAt",
]

item_fields: List[Field] = [
Field(name="inventoryHistoryUrl", alias="inventory_history_url"),
Expand All @@ -1280,12 +1279,27 @@ def _quantities_query(self) -> Query:
Defines the `quantities` nested query.
"""

return Query(
return Field(
name="quantities",
arguments=[Argument(name="names", value=self.quantities_names_filter)],
fields=self.quantities_fields,
fields=[
"id",
"name",
"quantity",
"updatedAt",
],
)

def _get_inventory_levels_fields(self, filter_query: Optional[str] = None) -> List[Field]:
nested_fields = self.inventory_levels_fields + [self._quantities_query()]
return self.query_nodes + [
Field(
name="inventoryLevels",
arguments=[Argument(name="query", value=f'"{filter_query}"')],
fields=[Field(name="edges", fields=[Field(name="node", fields=nested_fields)])],
)
]

def _process_quantities(self, quantities: Iterable[MutableMapping[str, Any]] = None) -> Iterable[Mapping[str, Any]]:
if quantities:
for quantity in quantities:
Expand All @@ -1299,15 +1313,10 @@ def _process_quantities(self, quantities: Iterable[MutableMapping[str, Any]] = N
return []

def query(self, filter_query: Optional[str] = None) -> Query:
# construct the `quantities` query piece
quantities: List[Query] = [self._quantities_query()]
# build the nested query first with `filter_query` to have the incremental syncs
inventory_levels: List[Query] = [self.build("inventoryLevels", self.inventory_levels_fields + quantities, filter_query)]
# build the main query around previous
# return the constructed query operation
return self.build(
name=self.query_name,
edges=self.query_nodes + inventory_levels,
edges=self._get_inventory_levels_fields(filter_query),
# passing more query args for `locations` query
additional_query_args=self.locations_query_args,
)
Expand Down Expand Up @@ -2311,8 +2320,9 @@ def _merge_with_media(self, record_components: List[dict]) -> Optional[Iterable[
if BULK_PARENT_KEY in item:
item.pop(BULK_PARENT_KEY)

image_url = item.get("image", {}).get("url")
if image_url in url_map:
image = item.get("image", {})
image_url = image.get("url") if image else None
if image_url and image_url in url_map:
# Merge images into media
item.update(url_map.get(image_url))
# remove lefovers
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-shopify/source_shopify/transform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def _first_non_null_type(schema_types: List[str]) -> str:

@staticmethod
def _transform_number(value: Any):
return Decimal(value)
return float(Decimal(value))

@staticmethod
def _transform_string(value: Any):
Expand Down
10 changes: 9 additions & 1 deletion airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,9 @@ def product_images_jsonl_content_example():
{"__typename":"Image","id":"gid:\/\/shopify\/ProductImage\/111","height":280,"alt":"","src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/white-t-shirt.jpg?v=1673029759","url":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/white-t-shirt.jpg?v=1673029759","width":265,"__parentId":"gid:\/\/shopify\/Product\/123"}
{"__typename":"Product","id":"gid:\/\/shopify\/Product\/456"}
{"__typename":"MediaImage","createdAt":"2021-06-23T01:09:47Z","updatedAt":"2023-04-24T17:27:15Z","image":{"url":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587"},"__parentId":"gid:\/\/shopify\/Product\/456"}
{"__typename":"Image","id":"gid:\/\/shopify\/ProductImage\/222","height":1467,"alt":"updated_mon_24.04.2023","src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587","url":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587","width":2200,"__parentId":"gid:\/\/shopify\/Product\/456"}\n"""
{"__typename":"Image","id":"gid:\/\/shopify\/ProductImage\/222","height":1467,"alt":"updated_mon_24.04.2023","src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587","url":"https:\/\/cdn.shopify.com\/s\/files\/1\/0580\/3317\/6765\/products\/4-ounce-soy-candle.jpg?v=1624410587","width":2200,"__parentId":"gid:\/\/shopify\/Product\/456"}
{"__typename":"Product","id":"gid:\/\/shopify\/Product\/9062091161885"}
{"__typename":"MediaImage","createdAt":"2024-06-12T23:41:27Z","updatedAt":"2024-06-12T23:41:28Z","image":null,"__parentId":"gid:\/\/shopify\/Product\/9062091161885"}\n"""


@pytest.fixture
Expand Down Expand Up @@ -812,6 +814,12 @@ def product_images_response_expected_result():
"product_id": 456,
"shop_url": "test_shop",
},
{
"created_at": "2024-06-12T23:41:27+00:00",
"updated_at": "2024-06-12T23:41:28+00:00",
"image": None,
"shop_url": "test_shop"
},
]


Expand Down
4 changes: 2 additions & 2 deletions airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ def test_base_build_query(query_name, fields, filter_field, start, end, expected
fields=[
'__typename',
'id',
Query(
Field(
name="inventoryLevels",
arguments=[
Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""),
Expand All @@ -206,7 +206,7 @@ def test_base_build_query(query_name, fields, filter_field, start, end, expected
"deactivationAlert",
"updatedAt",
Field(name="item", fields=[Field(name="inventoryHistoryUrl", alias="inventory_history_url"), Field(name="id", alias="inventory_item_id"), Field(name="locationsCount", alias="locations_count", fields=["count"])]),
Query(
Field(
name="quantities",
arguments=[
Argument(name="names", value=['"available"', '"incoming"', '"committed"', '"damaged"', '"on_hand"', '"quality_control"', '"reserved"', '"safety_stock"'])
Expand Down
14 changes: 7 additions & 7 deletions airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ def test_enforcer_correct_type(transform_object, schema, checks):
"path": [
"total_discounts",
],
"expected_type": Decimal,
"expected_type": float,
},
],
),
Expand All @@ -144,7 +144,7 @@ def test_enforcer_correct_type(transform_object, schema, checks):
"path": [
"total_price",
],
"expected_type": Decimal,
"expected_type": float,
},
],
),
Expand All @@ -168,7 +168,7 @@ def test_enforcer_string_to_number(transform_object, schema, checks):
},
},
[
{"path": ["customer", "total_spent"], "expected_type": Decimal},
{"path": ["customer", "total_spent"], "expected_type": float},
],
)
],
Expand All @@ -194,7 +194,7 @@ def test_enforcer_nested_object(transform_object, schema, checks):
},
},
[
{"path": ["shipping_lines", 0, "price"], "expected_type": Decimal},
{"path": ["shipping_lines", 0, "price"], "expected_type": float},
],
),
(
Expand All @@ -219,7 +219,7 @@ def test_enforcer_nested_object(transform_object, schema, checks):
},
},
[
{"path": ["line_items", 0, "line_price"], "expected_type": Decimal},
{"path": ["line_items", 0, "line_price"], "expected_type": float},
],
),
],
Expand Down Expand Up @@ -247,8 +247,8 @@ def test_enforcer_nested_array(transform_object, schema, checks):
},
},
[
{"path": ["line_items", 0], "expected_type": Decimal},
{"path": ["line_items", 1], "expected_type": Decimal},
{"path": ["line_items", 0], "expected_type": float},
{"path": ["line_items", 1], "expected_type": float},
],
),
],
Expand Down
1 change: 1 addition & 0 deletions docs/integrations/sources/shopify.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ For all `Shopify GraphQL BULK` api requests these limitations are applied: https

| Version | Date | Pull Request | Subject |
| :------ |:-----------| :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 2.4.7 | 2024-06-27 | [40593](https://github.com/airbytehq/airbyte/pull/40593) | Use latest `CDK` version possible |
| 2.4.6 | 2024-06-26 | [40526](https://github.com/airbytehq/airbyte/pull/40526) | Made `BULK Job termination threshold` limit adjustable from `input configuration`, increased the default value to `1 hour`. |
| 2.4.5 | 2024-06-25 | [40484](https://github.com/airbytehq/airbyte/pull/40484) | Update dependencies |
| 2.4.4 | 2024-06-19 | [39594](https://github.com/airbytehq/airbyte/pull/39594) | Extended the `Discount Codes`, `Fulfillment Orders`, `Inventory Items`, `Inventory Levels`, `Products`, `Product Variants` and `Transactions` stream schemas |
Expand Down

0 comments on commit 42c68d1

Please sign in to comment.