-
Notifications
You must be signed in to change notification settings - Fork 4.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
🐛 Source Amazon Seller Partner: fix OAuth (#32550)
Co-authored-by: davydov-d <davydov-d@users.noreply.github.com>
- Loading branch information
Showing
8 changed files
with
199 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
79 changes: 79 additions & 0 deletions
79
...connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| # | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
| # | ||
|
|
||
|
|
||
| import logging | ||
| from typing import Any, List, Mapping | ||
|
|
||
| from airbyte_cdk.config_observation import create_connector_config_control_message | ||
| from airbyte_cdk.entrypoint import AirbyteEntrypoint | ||
| from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository | ||
|
|
||
| from .source import SourceAmazonSellerPartner | ||
|
|
||
| logger = logging.getLogger("airbyte_logger") | ||
|
|
||
|
|
||
| class MigrateAccountType: | ||
| """ | ||
| This class stands for migrating the config at runtime, | ||
| while providing the backward compatibility when falling back to the previous source version. | ||
| Specifically, starting from `2.0.1`, the `account_type` property becomes required. | ||
| For those connector configs that do not contain this key, the default value of `Seller` will be used. | ||
| Reverse operation is not needed as this field is ignored in previous versions of the connector. | ||
| """ | ||
|
|
||
| message_repository: MessageRepository = InMemoryMessageRepository() | ||
| migration_key: str = "account_type" | ||
|
|
||
| @classmethod | ||
| def _should_migrate(cls, config: Mapping[str, Any]) -> bool: | ||
| """ | ||
| This method determines whether config requires migration. | ||
| Returns: | ||
| > True, if the transformation is neccessary | ||
| > False, otherwise. | ||
| """ | ||
| return cls.migration_key not in config | ||
|
|
||
| @classmethod | ||
| def _populate_with_default_value(cls, config: Mapping[str, Any], source: SourceAmazonSellerPartner = None) -> Mapping[str, Any]: | ||
| config[cls.migration_key] = "Seller" | ||
| return config | ||
|
|
||
| @classmethod | ||
| def _modify_and_save(cls, config_path: str, source: SourceAmazonSellerPartner, config: Mapping[str, Any]) -> Mapping[str, Any]: | ||
| # modify the config | ||
| migrated_config = cls._populate_with_default_value(config, source) | ||
| # save the config | ||
| source.write_config(migrated_config, config_path) | ||
| # return modified config | ||
| return migrated_config | ||
|
|
||
| @classmethod | ||
| def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: | ||
| # add the Airbyte Control Message to message repo | ||
| cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) | ||
| # emit the Airbyte Control Message from message queue to stdout | ||
| for message in cls.message_repository.consume_queue(): | ||
| print(message.json(exclude_unset=True)) | ||
|
|
||
| @classmethod | ||
| def migrate(cls, args: List[str], source: SourceAmazonSellerPartner) -> None: | ||
| """ | ||
| This method checks the input args, should the config be migrated, | ||
| transform if neccessary and emit the CONTROL message. | ||
| """ | ||
| # get config path | ||
| config_path = AirbyteEntrypoint(source).extract_config(args) | ||
| # proceed only if `--config` arg is provided | ||
| if config_path: | ||
| # read the existing config | ||
| config = source.read_config(config_path) | ||
| # migration check | ||
| if cls._should_migrate(config): | ||
| cls._emit_control_message( | ||
| cls._modify_and_save(config_path, source, config), | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| # | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
| # | ||
|
|
||
|
|
||
| import json | ||
| from typing import Any, Mapping | ||
|
|
||
| from airbyte_cdk.models import OrchestratorType, Type | ||
| from airbyte_cdk.sources import Source | ||
| from source_amazon_seller_partner.config_migrations import MigrateAccountType | ||
| from source_amazon_seller_partner.source import SourceAmazonSellerPartner | ||
|
|
||
| CMD = "check" | ||
| TEST_NOT_MIGRATED_CONFIG_PATH = "unit_tests/test_migrations/not_migrated_config.json" | ||
| TEST_MIGRATED_CONFIG_PATH = "unit_tests/test_migrations/migrated_config.json" | ||
| SOURCE: Source = SourceAmazonSellerPartner() | ||
|
|
||
|
|
||
| def load_config(config_path: str = TEST_NOT_MIGRATED_CONFIG_PATH) -> Mapping[str, Any]: | ||
| with open(config_path, "r") as config: | ||
| return json.load(config) | ||
|
|
||
|
|
||
| def test_migrate_config(capsys): | ||
| config = load_config(TEST_NOT_MIGRATED_CONFIG_PATH) | ||
| assert "acount_type" not in config | ||
| migration_instance = MigrateAccountType() | ||
| migration_instance.migrate([CMD, "--config", TEST_NOT_MIGRATED_CONFIG_PATH], SOURCE) | ||
| control_msg = json.loads(capsys.readouterr().out) | ||
| assert control_msg["type"] == Type.CONTROL.value | ||
| assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value | ||
| migrated_config = control_msg["control"]["connectorConfig"]["config"] | ||
| assert migrated_config["account_type"] == "Seller" | ||
|
|
||
|
|
||
| def test_should_not_migrate(): | ||
| config = load_config(TEST_MIGRATED_CONFIG_PATH) | ||
| assert config["account_type"] | ||
| migration_instance = MigrateAccountType() | ||
| assert not migration_instance._should_migrate(config) |
9 changes: 9 additions & 0 deletions
9
...s/connectors/source-amazon-seller-partner/unit_tests/test_migrations/migrated_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| { | ||
| "refresh_token": "refresh_token", | ||
| "lwa_app_id": "amzn1.application-oa2-client.lwa_app_id", | ||
| "lwa_client_secret": "amzn1.oa2-cs.v1.lwa_client_secret", | ||
| "replication_start_date": "2022-09-01T00:00:00Z", | ||
| "aws_environment": "PRODUCTION", | ||
| "account_type": "Vendor", | ||
| "region": "US" | ||
| } |
8 changes: 8 additions & 0 deletions
8
...nnectors/source-amazon-seller-partner/unit_tests/test_migrations/not_migrated_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| { | ||
| "refresh_token": "refresh_token", | ||
| "lwa_app_id": "amzn1.application-oa2-client.lwa_app_id", | ||
| "lwa_client_secret": "amzn1.oa2-cs.v1.lwa_client_secret", | ||
| "replication_start_date": "2022-09-01T00:00:00Z", | ||
| "aws_environment": "PRODUCTION", | ||
| "region": "US" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters