From c6038c12f375c7daaf2f003613cdb550df5971a1 Mon Sep 17 00:00:00 2001 From: Arsen Losenko Date: Thu, 13 Oct 2022 14:28:15 +0300 Subject: [PATCH 1/8] Add unit tests for auth.py and part of tests for source.py --- .../source-shopify/unit_tests/test_auth.py | 48 +++++++++++++ .../source-shopify/unit_tests/test_source.py | 71 +++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py new file mode 100644 index 000000000000..f3e6d7700b6c --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py @@ -0,0 +1,48 @@ + +import pytest +from source_shopify.auth import NotImplementedAuth, ShopifyAuthenticator + +TEST_ACCESS_TOKEN = "test_access_token" +TEST_API_PASSWORD = "test_api_password" + +@pytest.fixture +def config_access_token(): + return {"credentials": {"access_token": TEST_ACCESS_TOKEN, "auth_method": "access_token"}} + + +@pytest.fixture +def config_api_password(): + return {"credentials": {"api_password": TEST_API_PASSWORD, "auth_method": "api_password"}} + +@pytest.fixture +def config_not_implemented_auth_method(): + return {"credentials": {"auth_method": "not_implemented_auth_method"}} + + +@pytest.fixture +def expected_auth_header_access_token(): + return {"X-Shopify-Access-Token": TEST_ACCESS_TOKEN} + + +@pytest.fixture +def expected_auth_header_api_password(): + return {"X-Shopify-Access-Token": TEST_API_PASSWORD} + + +def test_shopify_authenticator_access_token(config_access_token, expected_auth_header_access_token): + authenticator = ShopifyAuthenticator(config=config_access_token) + assert authenticator.get_auth_header() == expected_auth_header_access_token + + +def test_shopify_authenticator_api_password(config_api_password, expected_auth_header_api_password): + authenticator = ShopifyAuthenticator(config=config_api_password) + assert authenticator.get_auth_header() == expected_auth_header_api_password + + +def test_raises_notimplemented_auth(config_not_implemented_auth_method): + authenticator = ShopifyAuthenticator(config=(config_not_implemented_auth_method)) + with pytest.raises(NotImplementedAuth) as not_implemented_exc: + print(not_implemented_exc) + authenticator.get_auth_header() + + diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py new file mode 100644 index 000000000000..a1363de9aec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -0,0 +1,71 @@ +import pytest +from source_shopify.auth import ShopifyAuthenticator +from source_shopify.source import ( + AbandonedCheckouts, + Collects, + CustomCollections, + Customers, + DiscountCodes, + DraftOrders, + FulfillmentOrders, + Fulfillments, + InventoryLevels, + Locations, + Metafields, + OrderRefunds, + OrderRisks, + Orders, + Pages, + PriceRules, + Products, + Shop, + TenderTransactions, + Transactions, +) + + +@pytest.fixture +def test_config(basic_config): + basic_config["start_date"] = "2020-11-01" + basic_config["authenticator"] = ShopifyAuthenticator(basic_config) + return basic_config + + +@pytest.mark.parametrize( + "stream,expected_path", + [ + (Customers, "customers.json"), + (Orders, "orders.json"), + (DraftOrders, "draft_orders.json"), + (Products, "products.json"), + (AbandonedCheckouts, "checkouts.json"), + (Metafields, "metafields.json"), + (Collects, "collects.json"), + (TenderTransactions, "tender_transactions.json"), + (Pages, "pages.json"), + (PriceRules, "price_rules.json"), + (Locations, "locations.json"), + (Shop, "shop.json"), + (CustomCollections, "custom_collections.json"), + ], +) +def test_customers_path(stream, expected_path, test_config): + stream = stream(test_config) + assert stream.path() == expected_path + + +@pytest.mark.parametrize( + "stream,stream_slice,expected_path", + [ + (OrderRefunds, {"order_id": 12345}, "orders/12345/refunds.json"), + (OrderRisks, {"order_id": 12345}, "orders/12345/risks.json"), + (Transactions, {"order_id": 12345}, "orders/12345/transactions.json"), + (DiscountCodes, {"price_rule_id": 12345}, "price_rules/12345/discount_codes.json"), + (InventoryLevels, {"location_id": 12345}, "locations/12345/inventory_levels.json"), + (FulfillmentOrders, {"order_id": 12345}, "orders/12345/fulfillment_orders.json"), + (Fulfillments, {"order_id": 12345}, "orders/12345/fulfillments.json"), + ], +) +def test_customers_path_with_stream_slice_param(stream, stream_slice, expected_path, test_config): + stream = stream(test_config) + assert stream.path(stream_slice) == expected_path From a52b324eb364124f6b45546f33bbf92197b4c92a Mon Sep 17 00:00:00 2001 From: Arsen Losenko Date: Fri, 14 Oct 2022 13:32:59 +0300 Subject: [PATCH 2/8] Add timeout to incremental tests --- .../connectors/source-shopify/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index cda71978b962..11cfcd227912 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -30,6 +30,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 3600 full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" From 9122063a34109072ce284686a3a4095dd556964e Mon Sep 17 00:00:00 2001 From: Arsen Losenko Date: Tue, 18 Oct 2022 13:10:49 +0300 Subject: [PATCH 3/8] Add more tests to source.py --- .../source-shopify/unit_tests/test_source.py | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index a1363de9aec6..45558128a4cd 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -1,3 +1,5 @@ +from unittest.mock import MagicMock, patch + import pytest from source_shopify.auth import ShopifyAuthenticator from source_shopify.source import ( @@ -19,13 +21,14 @@ PriceRules, Products, Shop, + SourceShopify, TenderTransactions, Transactions, ) @pytest.fixture -def test_config(basic_config): +def config(basic_config): basic_config["start_date"] = "2020-11-01" basic_config["authenticator"] = ShopifyAuthenticator(basic_config) return basic_config @@ -49,8 +52,8 @@ def test_config(basic_config): (CustomCollections, "custom_collections.json"), ], ) -def test_customers_path(stream, expected_path, test_config): - stream = stream(test_config) +def test_customers_path(stream, expected_path, config): + stream = stream(config) assert stream.path() == expected_path @@ -66,6 +69,30 @@ def test_customers_path(stream, expected_path, test_config): (Fulfillments, {"order_id": 12345}, "orders/12345/fulfillments.json"), ], ) -def test_customers_path_with_stream_slice_param(stream, stream_slice, expected_path, test_config): - stream = stream(test_config) +def test_customers_path_with_stream_slice_param(stream, stream_slice, expected_path, config): + stream = stream(config) assert stream.path(stream_slice) == expected_path + + +def test_check_connection(config, mocker): + mocker.patch("source_shopify.source.Shop.read_records", return_value=[{"id": 1}]) + source = SourceShopify() + logger_mock = MagicMock() + assert source.check_connection(logger_mock, config) == (True, None) + + +# def test_read_records(config, mocker): +# records = [{"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "2022-10-10T06:21:53-07:00"}}] +# stream_slice = records[0] +# stream = OrderRefunds(config) +# mocker.patch("source_shopify.source.IncrementalShopifyStream.read_records", return_value=records) +# assert next(stream.read_records(stream_slice=stream_slice)) == records[0] + + +# def test_get_updated_state(config, mocker): +# current_stream_state = [{"created_at": "2022-10-10T06:21:53-07:00"}] +# latest_record = {"created_at": "2022-10-10T06:22:53-07:00"} +# parent_state = {"orders": {"updated_at": "2022-10-10T06:21:53-07:00"}} +# updated_state = [{"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "2022-10-10T06:21:53-07:00"}}] +# stream = OrderRefunds(config) +# assert stream.get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) == updated_state From 3777ffcfd7b9e1118e8e6c02b2cfe8d5f94c4df4 Mon Sep 17 00:00:00 2001 From: Arsen Losenko Date: Fri, 21 Oct 2022 14:47:36 +0300 Subject: [PATCH 4/8] Update test_read_records and test_get_updated_state --- .../source-shopify/unit_tests/test_source.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index 45558128a4cd..037755b23829 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -81,18 +81,22 @@ def test_check_connection(config, mocker): assert source.check_connection(logger_mock, config) == (True, None) -# def test_read_records(config, mocker): -# records = [{"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "2022-10-10T06:21:53-07:00"}}] -# stream_slice = records[0] -# stream = OrderRefunds(config) -# mocker.patch("source_shopify.source.IncrementalShopifyStream.read_records", return_value=records) -# assert next(stream.read_records(stream_slice=stream_slice)) == records[0] +def test_read_records(config, mocker): + records = [{"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "2022-10-10T06:21:53-07:00"}}] + stream_slice = records[0] + stream = OrderRefunds(config) + mocker.patch("source_shopify.source.IncrementalShopifyStream.read_records", return_value=records) + assert next(stream.read_records(stream_slice=stream_slice)) == records[0] -# def test_get_updated_state(config, mocker): -# current_stream_state = [{"created_at": "2022-10-10T06:21:53-07:00"}] -# latest_record = {"created_at": "2022-10-10T06:22:53-07:00"} -# parent_state = {"orders": {"updated_at": "2022-10-10T06:21:53-07:00"}} -# updated_state = [{"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "2022-10-10T06:21:53-07:00"}}] -# stream = OrderRefunds(config) -# assert stream.get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) == updated_state +def test_request_params(config): + stream = OrderRefunds(config) + assert stream.request_params(next_page_token={"next_page": 2}) == {"limit": 250, "next_page": 2} + + +def test_get_updated_state(config, mocker): + current_stream_state = {"created_at": ""} + latest_record = {"created_at": "2022-10-10T06:21:53-07:00"} + updated_state = {"created_at": "2022-10-10T06:21:53-07:00", "orders": None} + stream = OrderRefunds(config) + assert stream.get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) == updated_state From 99d0a256bf6f524ada37356155fea88917c37695 Mon Sep 17 00:00:00 2001 From: Arsen Losenko Date: Fri, 21 Oct 2022 15:10:57 +0300 Subject: [PATCH 5/8] Update test_request_params to reach 90% test coverage --- .../source-shopify/unit_tests/test_source.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index 037755b23829..4d3294ff0285 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -1,3 +1,4 @@ +from re import A from unittest.mock import MagicMock, patch import pytest @@ -89,12 +90,19 @@ def test_read_records(config, mocker): assert next(stream.read_records(stream_slice=stream_slice)) == records[0] -def test_request_params(config): - stream = OrderRefunds(config) - assert stream.request_params(next_page_token={"next_page": 2}) == {"limit": 250, "next_page": 2} +@pytest.mark.parametrize( + "stream, expected", + [ + (OrderRefunds, {"limit": 250}), + (Orders, {"limit": 250, "status": "any", "order": "updated_at asc", "updated_at_min": "2020-11-01"}), + (AbandonedCheckouts, {"limit": 250, "status": "any", "order": "updated_at asc", "updated_at_min": "2020-11-01"}), + ], +) +def test_request_params(config, stream, expected): + assert stream(config).request_params() == expected -def test_get_updated_state(config, mocker): +def test_get_updated_state(config): current_stream_state = {"created_at": ""} latest_record = {"created_at": "2022-10-10T06:21:53-07:00"} updated_state = {"created_at": "2022-10-10T06:21:53-07:00", "orders": None} From 195038dc577633324260c0f1551e5c2bb80aef31 Mon Sep 17 00:00:00 2001 From: Arsen Losenko Date: Mon, 24 Oct 2022 15:13:12 +0300 Subject: [PATCH 6/8] Fix flake warnings --- .../connectors/source-shopify/unit_tests/test_source.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index 4d3294ff0285..100452ddaa3f 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -1,5 +1,3 @@ -from re import A -from unittest.mock import MagicMock, patch import pytest from source_shopify.auth import ShopifyAuthenticator From c61cea6c46bc6960b7d76a08deefdf85a44cacd0 Mon Sep 17 00:00:00 2001 From: Arsen Losenko Date: Mon, 24 Oct 2022 15:24:07 +0300 Subject: [PATCH 7/8] Add missing import --- .../connectors/source-shopify/unit_tests/test_source.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index 100452ddaa3f..cbf314bb7496 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -1,4 +1,6 @@ +from unittest.mock import MagicMock + import pytest from source_shopify.auth import ShopifyAuthenticator from source_shopify.source import ( From 17678ea16bfd713cd7de01233597561a662e4213 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 24 Oct 2022 15:57:09 +0300 Subject: [PATCH 8/8] updated tests --- .../source-shopify/unit_tests/test_auth.py | 8 ++- .../source-shopify/unit_tests/test_source.py | 72 ++++++++++++++----- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py index f3e6d7700b6c..f14a59829166 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + import pytest from source_shopify.auth import NotImplementedAuth, ShopifyAuthenticator @@ -5,6 +9,7 @@ TEST_ACCESS_TOKEN = "test_access_token" TEST_API_PASSWORD = "test_api_password" + @pytest.fixture def config_access_token(): return {"credentials": {"access_token": TEST_ACCESS_TOKEN, "auth_method": "access_token"}} @@ -14,6 +19,7 @@ def config_access_token(): def config_api_password(): return {"credentials": {"api_password": TEST_API_PASSWORD, "auth_method": "api_password"}} + @pytest.fixture def config_not_implemented_auth_method(): return {"credentials": {"auth_method": "not_implemented_auth_method"}} @@ -44,5 +50,3 @@ def test_raises_notimplemented_auth(config_not_implemented_auth_method): with pytest.raises(NotImplementedAuth) as not_implemented_exc: print(not_implemented_exc) authenticator.get_auth_header() - - diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index cbf314bb7496..d57662efc6e8 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + from unittest.mock import MagicMock @@ -5,6 +9,8 @@ from source_shopify.auth import ShopifyAuthenticator from source_shopify.source import ( AbandonedCheckouts, + Articles, + Blogs, Collects, CustomCollections, Customers, @@ -14,13 +20,26 @@ Fulfillments, InventoryLevels, Locations, - Metafields, + MetafieldArticles, + MetafieldBlogs, + MetafieldCollections, + MetafieldCustomers, + MetafieldDraftOrders, + MetafieldLocations, + MetafieldOrders, + MetafieldPages, + MetafieldProducts, + MetafieldProductVariants, + MetafieldShops, + MetafieldSmartCollections, OrderRefunds, OrderRisks, Orders, Pages, PriceRules, + ProductImages, Products, + ProductVariants, Shop, SourceShopify, TenderTransactions, @@ -36,26 +55,45 @@ def config(basic_config): @pytest.mark.parametrize( - "stream,expected_path", + "stream,stream_slice,expected_path", [ - (Customers, "customers.json"), - (Orders, "orders.json"), - (DraftOrders, "draft_orders.json"), - (Products, "products.json"), - (AbandonedCheckouts, "checkouts.json"), - (Metafields, "metafields.json"), - (Collects, "collects.json"), - (TenderTransactions, "tender_transactions.json"), - (Pages, "pages.json"), - (PriceRules, "price_rules.json"), - (Locations, "locations.json"), - (Shop, "shop.json"), - (CustomCollections, "custom_collections.json"), + (Articles, None, "articles.json"), + (Blogs, None, "blogs.json"), + (MetafieldBlogs, {"id": 123}, "blogs/123/metafields.json"), + (MetafieldArticles, {"id": 123}, "articles/123/metafields.json"), + (MetafieldCustomers, {"id": 123}, "customers/123/metafields.json"), + (MetafieldOrders, {"id": 123}, "orders/123/metafields.json"), + (MetafieldDraftOrders, {"id": 123}, "draft_orders/123/metafields.json"), + (MetafieldProducts, {"id": 123}, "products/123/metafields.json"), + (MetafieldProductVariants, {"variants": 123}, "variants/123/metafields.json"), + (MetafieldSmartCollections, {"id": 123}, "smart_collections/123/metafields.json"), + (MetafieldCollections, {"collection_id": 123}, "collections/123/metafields.json"), + (MetafieldPages, {"id": 123}, "pages/123/metafields.json"), + (MetafieldLocations, {"id": 123}, "locations/123/metafields.json"), + (MetafieldShops, None, "metafields.json"), + (ProductImages, {"product_id": 123}, "products/123/images.json"), + (ProductVariants, {"product_id": 123}, "products/123/variants.json"), + (Customers, None, "customers.json"), + (Orders, None, "orders.json"), + (DraftOrders, None, "draft_orders.json"), + (Products, None, "products.json"), + (AbandonedCheckouts, None, "checkouts.json"), + (Collects, None, "collects.json"), + (TenderTransactions, None, "tender_transactions.json"), + (Pages, None, "pages.json"), + (PriceRules, None, "price_rules.json"), + (Locations, None, "locations.json"), + (Shop, None, "shop.json"), + (CustomCollections, None, "custom_collections.json"), ], ) -def test_customers_path(stream, expected_path, config): +def test_customers_path(stream, stream_slice, expected_path, config): stream = stream(config) - assert stream.path() == expected_path + if stream_slice: + result = stream.path(stream_slice) + else: + result = stream.path() + assert result == expected_path @pytest.mark.parametrize(