🎉 Amazon SP extra endpoint support

airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.0
+LABEL io.airbyte.version=0.2.1
 LABEL io.airbyte.name=airbyte/source-amazon-seller-partner

airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml

@@ -3,22 +3,38 @@ tests:
   spec:
     - spec_path: "source_amazon_seller_partner/spec.json"
   connection:
-   - config_path: "secrets/config.json"
-     status: "succeed"
-   - config_path: "integration_tests/invalid_config.json"
-     status: "failed"
+    - config_path: "secrets/config.json"
+      status: "succeed"
+      timeout_seconds: 60
+    - config_path: "integration_tests/invalid_config.json"
+      status: "failed"
+      timeout_seconds: 60
   discovery:
-   - config_path: "secrets/config.json"
-  basic_read:
-   - config_path: "secrets/config.json"
-     configured_catalog_path: "integration_tests/configured_catalog_no_orders.json"
-     empty_streams: []
-  incremental:
-   - config_path: "secrets/config.json"
-     configured_catalog_path: "integration_tests/configured_catalog_no_orders.json"
-     future_state_path: "integration_tests/abnormal_state.json"
-     cursor_paths:
-       GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL: ["createdTime"]
-  full_refresh:
-   - config_path: "secrets/config.json"
-     configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config.json"
+# TODO: uncomment when at least one record exist
+#  basic_read:
+#    - config_path: "secrets/config.json"
+#      configured_catalog_path: "integration_tests/configured_catalog.json"
+#      empty_streams:
+#        [
+#          "Orders",
+#          "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
+#          "GET_MERCHANT_LISTINGS_ALL_DATA",
+#          "GET_FBA_INVENTORY_AGED_DATA",
+#          "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL",
+#          "GET_FLAT_FILE_OPEN_LISTINGS_DATA",
+#          "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
+#          "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
+#          "GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT",
+#          "VendorDirectFulfillmentShipping",
+#        ]
+# TODO: uncomment when Orders (or any other incremental) stream is filled with data
+#  incremental:
+#   - config_path: "secrets/config.json"
+#     configured_catalog_path: "integration_tests/configured_catalog.json"
+#     future_state_path: "integration_tests/future_state.json"
+#     cursor_paths:
+#       Orders: ["LastUpdateDate"]
+#  full_refresh:
+#   - config_path: "secrets/config.json"
+#     configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog.json

@@ -16,37 +16,82 @@
       "stream": {
         "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdTime"]
+        "supported_sync_modes": ["full_refresh"]
       },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["createdTime"]
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
     },
     {
       "stream": {
         "name": "GET_MERCHANT_LISTINGS_ALL_DATA",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdTime"]
+        "supported_sync_modes": ["full_refresh"]
       },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["createdTime"]
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
     },
     {
       "stream": {
         "name": "GET_FBA_INVENTORY_AGED_DATA",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdTime"]
+        "supported_sync_modes": ["full_refresh"]
       },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["createdTime"]
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_FLAT_FILE_OPEN_LISTINGS_DATA",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "VendorDirectFulfillmentShipping",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
     }
   ]
 }

airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_empty_streams.json

@@ -0,0 +1,49 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_FBA_INVENTORY_AGED_DATA",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}

airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json

@@ -0,0 +1,5 @@
+{
+  "Orders": {
+    "LastUpdateDate": "2121-07-01T00:00:00Z"
+  }
+}

airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json

@@ -10,5 +10,20 @@
   },
   "GET_FBA_INVENTORY_AGED_DATA": {
     "createdTime": "2021-07-01T00:00:00Z"
+  },
+  "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL": {
+    "createdTime": "2021-07-01T00:00:00Z"
+  },
+  "GET_FLAT_FILE_OPEN_LISTINGS_DATA": {
+    "createdTime": "2021-07-01T00:00:00Z"
+  },
+  "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA": {
+    "createdTime": "2021-07-01T00:00:00Z"
+  },
+  "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA": {
+    "createdTime": "2021-07-01T00:00:00Z"
+  },
+  "GET_VENDOR_INVENTORY_HEALTH_REPORT": {
+    "createdTime": "2021-07-01T00:00:00Z"
   }
 }

airbyte-integrations/connectors/source-amazon-seller-partner/setup.py

@@ -25,7 +25,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "boto3~=1.16", "pendulum~=2.1"]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "boto3~=1.16", "pendulum~=2.1", "pycryptodome~=3.10"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",

airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py

@@ -26,12 +26,12 @@
 import hmac
 import urllib.parse
 from typing import Any, Mapping
+from urllib.parse import urlparse
 
 import pendulum
 import requests
 from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator
 from requests.auth import AuthBase
-from requests.compat import urlparse
 
 
 class AWSAuthenticator(Oauth2Authenticator):